From fe8d290c171f02aa90906df579f8e579e32cedef Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Mon, 29 Sep 2014 17:33:37 -0400 Subject: [PATCH 1/6] Implemented cached resolution (minigraph caching) --- ivy/src/main/scala/sbt/Ivy.scala | 24 +- ivy/src/main/scala/sbt/IvyActions.scala | 29 +- ivy/src/main/scala/sbt/IvyRetrieve.scala | 24 +- ivy/src/main/scala/sbt/LogicalClock.scala | 16 + ivy/src/main/scala/sbt/UpdateOptions.scala | 20 +- ivy/src/main/scala/sbt/UpdateReport.scala | 5 +- .../CachedResolutionResolveEngine.scala | 411 ++++++++++++++++++ .../ivyint/ConsolidatedResolveEngine.scala | 120 ----- ivy/src/test/scala/BaseIvySpecification.scala | 26 +- ivy/src/test/scala/CachedResolutionSpec.scala | 51 +++ .../actions/src/main/scala/sbt/CacheIvy.scala | 3 +- main/src/main/scala/sbt/BuildPaths.scala | 6 + main/src/main/scala/sbt/Defaults.scala | 17 +- project/Sbt.scala | 2 +- project/Util.scala | 3 + .../consolidated-resolution/multi.sbt | 24 +- .../consolidated-resolution/test | 2 +- 17 files changed, 602 insertions(+), 181 deletions(-) create mode 100644 ivy/src/main/scala/sbt/LogicalClock.scala create mode 100644 ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala delete mode 100644 ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala create mode 100644 ivy/src/test/scala/CachedResolutionSpec.scala diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index cc39ab42e..db8d8ed1e 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -4,7 +4,7 @@ package sbt import Resolver.PluginPattern -import ivyint.{ ConsolidatedResolveEngine, ConsolidatedResolveCache } +import ivyint.{ CachedResolutionResolveEngine, CachedResolutionResolveCache } import java.io.File import java.net.URI @@ -84,7 +84,7 @@ final class IvySbt(val configuration: IvyConfiguration) { } is } - private lazy val ivy: Ivy = + private[sbt] def mkIvy: Ivy = { val i = new Ivy() { private val loggerEngine = new SbtMessageLoggerEngine @@ -94,9 +94,9 @@ final class IvySbt(val configuration: IvyConfiguration) { // We inject the deps we need before we can hook our resolve engine. setSortEngine(new SortEngine(getSettings)) setEventManager(new EventManager()) - if (configuration.updateOptions.consolidatedResolution) { - setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine) with ConsolidatedResolveEngine { - val consolidatedResolveCache = IvySbt.consolidatedResolveCache + if (configuration.updateOptions.cachedResolution) { + setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine) with CachedResolutionResolveEngine { + val cachedResolutionResolveCache = IvySbt.cachedResolutionResolveCache val projectResolver = prOpt }) } else setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine)) @@ -108,6 +108,8 @@ final class IvySbt(val configuration: IvyConfiguration) { i.getLoggerEngine.pushLogger(new IvyLoggerInterface(configuration.log)) i } + + private lazy val ivy: Ivy = mkIvy // Must be the same file as is used in Update in the launcher private lazy val ivyLockFile = new File(settings.getDefaultIvyUserDir, ".sbt.ivy.lock") /** ========== End Configuration/Setup ============*/ @@ -130,14 +132,14 @@ final class IvySbt(val configuration: IvyConfiguration) { } /** - * Cleans consolidated resolution cache. + * Cleans cached resolution cache. * @param md - module descriptor of the original Ivy graph. */ - private[sbt] def cleanConsolidatedResolutionCache(md: ModuleDescriptor, log: Logger): Unit = + private[sbt] def cleanCachedResolutionCache(md: ModuleDescriptor, log: Logger): Unit = withIvy(log) { i => val prOpt = Option(i.getSettings.getResolver(ProjectResolver.InterProject)) map { case pr: ProjectResolver => pr } - if (configuration.updateOptions.consolidatedResolution) { - IvySbt.consolidatedResolveCache.clean(md, prOpt) + if (configuration.updateOptions.cachedResolution) { + IvySbt.cachedResolutionResolveCache.clean(md, prOpt) } } @@ -242,7 +244,7 @@ private[sbt] object IvySbt { val DefaultIvyFilename = "ivy.xml" val DefaultMavenFilename = "pom.xml" val DefaultChecksums = Seq("sha1", "md5") - private[sbt] val consolidatedResolveCache: ConsolidatedResolveCache = new ConsolidatedResolveCache() + private[sbt] val cachedResolutionResolveCache: CachedResolutionResolveCache = new CachedResolutionResolveCache() def defaultIvyFile(project: File) = new File(project, DefaultIvyFilename) def defaultIvyConfiguration(project: File) = new File(project, DefaultIvyConfigFilename) @@ -270,6 +272,8 @@ private[sbt] object IvySbt { val mainChain = makeChain("Default", "sbt-chain", resolvers) settings.setDefaultResolver(mainChain.getName) } + private[sbt] def isChanging(module: ModuleID): Boolean = + module.revision endsWith "-SNAPSHOT" private[sbt] def isChanging(mrid: ModuleRevisionId): Boolean = mrid.getRevision endsWith "-SNAPSHOT" def resolverChain(name: String, resolvers: Seq[Resolver], localOnly: Boolean, settings: IvySettings, log: Logger): DependencyResolver = diff --git a/ivy/src/main/scala/sbt/IvyActions.scala b/ivy/src/main/scala/sbt/IvyActions.scala index 4d69ae2c7..0d12d5b06 100644 --- a/ivy/src/main/scala/sbt/IvyActions.scala +++ b/ivy/src/main/scala/sbt/IvyActions.scala @@ -6,6 +6,7 @@ package sbt import java.io.File import scala.xml.{ Node => XNode, NodeSeq } import collection.mutable +import ivyint.CachedResolutionResolveEngine import org.apache.ivy.Ivy import org.apache.ivy.core.{ IvyPatternHelper, LogOptions } @@ -69,12 +70,12 @@ object IvyActions { } /** - * Cleans the consolidated resolution cache, if any. + * Cleans the cached resolution cache, if any. * This is called by clean. */ - private[sbt] def cleanConsolidatedResolutionCache(module: IvySbt#Module, log: Logger): Unit = + private[sbt] def cleanCachedResolutionCache(module: IvySbt#Module, log: Logger): Unit = module.withModule(log) { (ivy, md, default) => - module.owner.cleanConsolidatedResolutionCache(md, log) + module.owner.cleanCachedResolutionCache(md, log) } /** Creates a Maven pom from the given Ivy configuration*/ @@ -141,7 +142,7 @@ object IvyActions { */ @deprecated("Use updateEither instead.", "0.13.6") def update(module: IvySbt#Module, configuration: UpdateConfiguration, log: Logger): UpdateReport = - updateEither(module, configuration, UnresolvedWarningConfiguration(), log) match { + updateEither(module, configuration, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) match { case Right(r) => r case Left(w) => throw w.resolveException @@ -151,9 +152,25 @@ object IvyActions { * Resolves and retrieves dependencies. 'ivyConfig' is used to produce an Ivy file and configuration. * 'updateConfig' configures the actual resolution and retrieval process. */ - def updateEither(module: IvySbt#Module, configuration: UpdateConfiguration, - uwconfig: UnresolvedWarningConfiguration, log: Logger): Either[UnresolvedWarning, UpdateReport] = + private[sbt] def updateEither(module: IvySbt#Module, configuration: UpdateConfiguration, + uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): Either[UnresolvedWarning, UpdateReport] = module.withModule(log) { + case (ivy, md, default) if module.owner.configuration.updateOptions.cachedResolution => + ivy.getResolveEngine match { + case x: CachedResolutionResolveEngine => + val resolveOptions = new ResolveOptions + val resolveId = ResolveOptions.getDefaultResolveId(md) + resolveOptions.setResolveId(resolveId) + x.customResolve(md, logicalClock, resolveOptions, depDir getOrElse { sys.error("dependency base directory is not specified") }, log) match { + case Left(x) => + Left(UnresolvedWarning(x, uwconfig)) + case Right(uReport) => + configuration.retrieve match { + case Some(rConf) => Right(retrieve(ivy, uReport, rConf)) + case None => Right(uReport) + } + } + } case (ivy, md, default) => val (report, err) = resolve(configuration.logging)(ivy, md, default) err match { diff --git a/ivy/src/main/scala/sbt/IvyRetrieve.scala b/ivy/src/main/scala/sbt/IvyRetrieve.scala index 5f07963bd..bb15c68df 100644 --- a/ivy/src/main/scala/sbt/IvyRetrieve.scala +++ b/ivy/src/main/scala/sbt/IvyRetrieve.scala @@ -76,8 +76,12 @@ object IvyRetrieve { val callerConfigurations = caller.getCallerConfigurations.toArray.toVector collect { case x if nonEmptyString(x).isDefined => x } - val extraAttributes = toExtraAttributes(caller.getDependencyDescriptor.getExtraAttributes) - new Caller(m, callerConfigurations, extraAttributes) + val ddOpt = Option(caller.getDependencyDescriptor) + val (extraAttributes, isForce, isChanging, isTransitive) = ddOpt match { + case Some(dd) => (toExtraAttributes(dd.getExtraAttributes), dd.isForce, dd.isChanging, dd.isTransitive) + case None => (Map.empty[String, String], false, false, true) + } + new Caller(m, callerConfigurations, extraAttributes, isForce, isChanging, isTransitive) } val revId = dep.getResolvedId val moduleId = toModuleID(revId) @@ -160,19 +164,11 @@ object IvyRetrieve { */ def findPath(target: IvyNode, from: ModuleRevisionId): List[IvyNode] = { def doFindPath(current: IvyNode, path: List[IvyNode]): List[IvyNode] = { - val callers = current.getAllRealCallers.toList - // Ivy actually returns non-direct callers here. + // Ivy actually returns mix of direct and non-direct callers here. // that's why we have to calculate all possible paths below and pick the longest path. - val directCallers = callers filter { caller => - val md = caller.getModuleDescriptor - val dd = md.getDependencies.toList find { dd => - (dd.getDependencyRevisionId == current.getId) && - (dd.getParentRevisionId == caller.getModuleRevisionId) - } - dd.isDefined - } - val directCallersRevId = (directCallers map { _.getModuleRevisionId }).distinct - val paths: List[List[IvyNode]] = ((directCallersRevId map { revId => + val callers = current.getAllRealCallers.toList + val callersRevId = (callers map { _.getModuleRevisionId }).distinct + val paths: List[List[IvyNode]] = ((callersRevId map { revId => val node = current.findNode(revId) if (revId == from) node :: path else if (node == node.getRoot) Nil diff --git a/ivy/src/main/scala/sbt/LogicalClock.scala b/ivy/src/main/scala/sbt/LogicalClock.scala new file mode 100644 index 000000000..1056f6560 --- /dev/null +++ b/ivy/src/main/scala/sbt/LogicalClock.scala @@ -0,0 +1,16 @@ +package sbt + +/** + * Represents a logical time point for dependency resolution. + * This is used to cache dependencies across subproject resolution which may change over time. + */ +trait LogicalClock { + def toString: String +} + +object LogicalClock { + def apply(x: String): LogicalClock = new LogicalClock { + override def toString: String = x + } + def unknown: LogicalClock = apply("unknown") +} diff --git a/ivy/src/main/scala/sbt/UpdateOptions.scala b/ivy/src/main/scala/sbt/UpdateOptions.scala index d3bf28bfc..1d8ad1aec 100644 --- a/ivy/src/main/scala/sbt/UpdateOptions.scala +++ b/ivy/src/main/scala/sbt/UpdateOptions.scala @@ -15,22 +15,31 @@ final class UpdateOptions private[sbt] ( /** If set to true, check all resolvers for snapshots. */ val latestSnapshots: Boolean, /** If set to true, use consolidated resolution. */ - val consolidatedResolution: Boolean) { + val consolidatedResolution: Boolean, + /** If set to true, use cached resolution. */ + val cachedResolution: Boolean) { def withCircularDependencyLevel(circularDependencyLevel: CircularDependencyLevel): UpdateOptions = copy(circularDependencyLevel = circularDependencyLevel) def withLatestSnapshots(latestSnapshots: Boolean): UpdateOptions = copy(latestSnapshots = latestSnapshots) + @deprecated("Use withCachedResolution instead.", "0.13.7") def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions = - copy(consolidatedResolution = consolidatedResolution) + copy(consolidatedResolution = consolidatedResolution, + cachedResolution = consolidatedResolution) + def withCachedResolution(cachedResoluton: Boolean): UpdateOptions = + copy(cachedResolution = cachedResoluton, + consolidatedResolution = cachedResolution) private[sbt] def copy( circularDependencyLevel: CircularDependencyLevel = this.circularDependencyLevel, latestSnapshots: Boolean = this.latestSnapshots, - consolidatedResolution: Boolean = this.consolidatedResolution): UpdateOptions = + consolidatedResolution: Boolean = this.consolidatedResolution, + cachedResolution: Boolean = this.cachedResolution): UpdateOptions = new UpdateOptions(circularDependencyLevel, latestSnapshots, - consolidatedResolution) + consolidatedResolution, + cachedResolution) } object UpdateOptions { @@ -38,5 +47,6 @@ object UpdateOptions { new UpdateOptions( circularDependencyLevel = CircularDependencyLevel.Warn, latestSnapshots = true, - consolidatedResolution = false) + consolidatedResolution = false, + cachedResolution = false) } diff --git a/ivy/src/main/scala/sbt/UpdateReport.scala b/ivy/src/main/scala/sbt/UpdateReport.scala index 7e04d5135..443c84ed3 100644 --- a/ivy/src/main/scala/sbt/UpdateReport.scala +++ b/ivy/src/main/scala/sbt/UpdateReport.scala @@ -187,7 +187,10 @@ object ModuleReport { final class Caller( val caller: ModuleID, val callerConfigurations: Seq[String], - val callerExtraAttributes: Map[String, String]) { + val callerExtraAttributes: Map[String, String], + val isForceDependency: Boolean, + val isChangingDependency: Boolean, + val isTransitiveDependency: Boolean) { override def toString: String = s"$caller" } diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala new file mode 100644 index 000000000..9db9343bb --- /dev/null +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -0,0 +1,411 @@ +package sbt +package ivyint + +import java.util.Date +import java.net.URL +import java.io.File +import collection.concurrent +import collection.immutable.ListMap +import org.apache.ivy.core +import core.resolve._ +import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } +import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport } +import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration } +import core.{ IvyPatternHelper, LogOptions } +import org.apache.ivy.util.Message +import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } +import org.json4s._ + +private[sbt] object CachedResolutionResolveCache { + def createID(organization: String, name: String, revision: String) = + ModuleRevisionId.newInstance(organization, name, revision) + def sbtOrgTemp = "org.scala-sbt.temp" +} + +class CachedResolutionResolveCache() { + import CachedResolutionResolveCache._ + val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap() + val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap() + val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap() + val directDependencyCache: concurrent.Map[ModuleRevisionId, Vector[DependencyDescriptor]] = concurrent.TrieMap() + val conflictCache: concurrent.Map[(ModuleID, ModuleID), (Vector[ModuleID], Vector[ModuleID], String)] = concurrent.TrieMap() + val maxConflictCacheSize: Int = 10000 + + def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = { + val mrid0 = md0.getModuleRevisionId + val mds = + if (mrid0.getOrganisation == sbtOrgTemp) Vector(md0) + else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 } + mds foreach { md => + updateReportCache.remove(md.getModuleRevisionId) + // todo: backport this + directDependencyCache.remove(md.getModuleRevisionId) + } + } + def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] = + directDependencyCache.getOrElseUpdate(md0.getModuleRevisionId, md0.getDependencies.toVector) + + def buildArtificialModuleDescriptors(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] = + { + def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] = + prOpt map { + _.getModuleDescriptor(dep.getDependencyRevisionId) match { + case Some(internal) => directDependencies(internal) flatMap expandInternalDeps + case _ => Vector(dep) + } + } getOrElse Vector(dep) + val expanded = directDependencies(md0) flatMap expandInternalDeps + val rootModuleConfigs = md0.getConfigurations.toVector + expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, prOpt) } + } + def isDependencyChanging(dd: DependencyDescriptor): Boolean = + dd.isChanging || IvySbt.isChanging(dd.getDependencyRevisionId) + def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) = + { + val mrid = dd.getDependencyRevisionId + val confMap = (dd.getModuleConfigurations map { conf => + conf + "->(" + dd.getDependencyConfigurations(conf).mkString(",") + ")" + }) + val depsString = mrid.toString + ";" + confMap.mkString(";") + val sha1 = Hash.toHex(Hash(depsString)) + val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor { + def targetModuleRevisionId: ModuleRevisionId = mrid + } + for { + conf <- rootModuleConfigs + } yield md1.addConfiguration(conf) + md1.addDependency(dd) + (md1, isDependencyChanging(dd)) + } + def getOrElseUpdateMiniGraph(md: ModuleDescriptor, changing0: Boolean, logicalClock: LogicalClock, miniGraphPath: File, cachedDescriptor: File, log: Logger)(f: => Either[ResolveException, UpdateReport]): Either[ResolveException, UpdateReport] = + { + import Path._ + val mrid = md.getResolvedModuleRevisionId + val (pathOrg, pathName, pathRevision) = md match { + case x: ArtificialModuleDescriptor => + val tmrid = x.targetModuleRevisionId + (tmrid.getOrganisation, tmrid.getName, tmrid.getRevision + "_" + mrid.getName) + case _ => + (mrid.getOrganisation, mrid.getName, mrid.getRevision) + } + val staticGraphDirectory = miniGraphPath / "static" + val dynamicGraphDirectory = miniGraphPath / "dynamic" + val staticGraphPath = staticGraphDirectory / pathOrg / pathName / pathRevision / "graphs" / "graph.json" + val dynamicGraphPath = dynamicGraphDirectory / logicalClock.toString / pathOrg / pathName / pathRevision / "graphs" / "graph.json" + def loadMiniGraphFromFile: Option[Either[ResolveException, UpdateReport]] = + (if (staticGraphPath.exists) Some(staticGraphPath) + else if (dynamicGraphPath.exists) Some(dynamicGraphPath) + else None) map { path => + log.debug(s"parsing ${path.getAbsolutePath.toString}") + val ur = parseUpdateReport(md, path, cachedDescriptor, log) + updateReportCache(md.getModuleRevisionId) = Right(ur) + Right(ur) + } + (updateReportCache.get(mrid) orElse loadMiniGraphFromFile) match { + case Some(result) => result + case None => + f match { + case Right(ur) => + val changing = changing0 || (ur.configurations exists { cr => + cr.details exists { oar => + oar.modules exists { mr => + IvySbt.isChanging(mr.module) || (mr.callers exists { _.isChangingDependency }) + } + } + }) + IO.createDirectory(miniGraphPath) + writeUpdateReport(ur, + if (changing) dynamicGraphPath + else staticGraphPath) + // don't cache dynamic graphs in memory. + if (!changing) { + updateReportCache(md.getModuleRevisionId) = Right(ur) + } + Right(ur) + case Left(re) => + if (!changing0) { + updateReportCache(md.getModuleRevisionId) = Left(re) + } + Left(re) + } + } + } + def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = + { + import org.json4s._ + implicit val formats = native.Serialization.formats(NoTypeHints) + + new ConfigurationSerializer + + new ArtifactSerializer + + new FileSerializer + try { + val json = jawn.support.json4s.Parser.parseFromFile(path) + fromLite(json.get.extract[UpdateReportLite], cachedDescriptor) + } catch { + case e: Throwable => + log.error("Unable to parse mini graph: " + path.toString) + throw e + } + } + def writeUpdateReport(ur: UpdateReport, graphPath: File): Unit = + { + implicit val formats = native.Serialization.formats(NoTypeHints) + + new ConfigurationSerializer + + new ArtifactSerializer + + new FileSerializer + import native.Serialization.write + println(graphPath.toString) + val str = write(toLite(ur)) + IO.write(graphPath, str, IO.utf8) + } + def toLite(ur: UpdateReport): UpdateReportLite = + UpdateReportLite(ur.configurations map { cr => + ConfigurationReportLite(cr.configuration, cr.details) + }) + def fromLite(lite: UpdateReportLite, cachedDescriptor: File): UpdateReport = + { + val stats = new UpdateStats(0L, 0L, 0L, false) + val configReports = lite.configurations map { cr => + val details = cr.details + val modules = details flatMap { + _.modules filter { mr => + !mr.evicted && mr.problem.isEmpty + } + } + val evicted = details flatMap { + _.modules filter { mr => + mr.evicted + } + } map { _.module } + new ConfigurationReport(cr.configuration, modules, details, evicted) + } + new UpdateReport(cachedDescriptor, configReports, stats) + } + def getOrElseUpdateConflict(cf0: ModuleID, cf1: ModuleID, conflicts: Vector[ModuleReport])(f: => (Vector[ModuleReport], Vector[ModuleReport], String)): (Vector[ModuleReport], Vector[ModuleReport]) = + { + def reconstructReports(surviving: Vector[ModuleID], evicted: Vector[ModuleID], mgr: String): (Vector[ModuleReport], Vector[ModuleReport]) = { + val moduleIdMap = Map(conflicts map { x => x.module -> x }: _*) + (surviving map moduleIdMap, evicted map moduleIdMap map { _.copy(evicted = true, evictedReason = Some(mgr.toString)) }) + } + (conflictCache get ((cf0, cf1))) match { + case Some((surviving, evicted, mgr)) => reconstructReports(surviving, evicted, mgr) + case _ => + (conflictCache get ((cf1, cf0))) match { + case Some((surviving, evicted, mgr)) => reconstructReports(surviving, evicted, mgr) + case _ => + val (surviving, evicted, mgr) = f + if (conflictCache.size > maxConflictCacheSize) { + conflictCache.remove(conflictCache.head._1) + } + conflictCache((cf0, cf1)) = (surviving map { _.module }, evicted map { _.module }, mgr) + (surviving, evicted) + } + } + } + class FileSerializer extends CustomSerializer[File](format => ( + { + case JString(s) => new File(s) + }, + { + case x: File => JString(x.toString) + } + )) + class ConfigurationSerializer extends CustomSerializer[Configuration](format => ( + { + case JString(s) => new Configuration(s) + }, + { + case x: Configuration => JString(x.name) + } + )) + class ArtifactSerializer extends CustomSerializer[Artifact](format => ( + { + case json: JValue => + implicit val fmt = format + Artifact( + (json \ "name").extract[String], + (json \ "type").extract[String], + (json \ "extension").extract[String], + (json \ "classifier").extract[Option[String]], + (json \ "configurations").extract[List[Configuration]], + (json \ "url").extract[Option[URL]], + (json \ "extraAttributes").extract[Map[String, String]] + ) + }, + { + case x: Artifact => + import DefaultJsonFormats.{ OptionWriter, StringWriter, mapWriter } + val optStr = implicitly[Writer[Option[String]]] + val mw = implicitly[Writer[Map[String, String]]] + JObject(JField("name", JString(x.name)) :: + JField("type", JString(x.`type`)) :: + JField("extension", JString(x.extension)) :: + JField("classifier", optStr.write(x.classifier)) :: + JField("configurations", JArray(x.configurations.toList map { x => JString(x.name) })) :: + JField("url", optStr.write(x.url map { _.toString })) :: + JField("extraAttributes", mw.write(x.extraAttributes)) :: + Nil) + } + )) +} + +private[sbt] trait ArtificialModuleDescriptor { self: DefaultModuleDescriptor => + def targetModuleRevisionId: ModuleRevisionId +} + +private[sbt] case class UpdateReportLite(configurations: Seq[ConfigurationReportLite]) +private[sbt] case class ConfigurationReportLite(configuration: String, details: Seq[OrganizationArtifactReport]) + +private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { + import CachedResolutionResolveCache._ + + private[sbt] def cachedResolutionResolveCache: CachedResolutionResolveCache + private[sbt] def projectResolver: Option[ProjectResolver] + + // Return sbt's UpdateReport. + def customResolve(md0: ModuleDescriptor, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { + import Path._ + val miniGraphPath = depDir / "module" + val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) + val cache = cachedResolutionResolveCache + val mds = cache.buildArtificialModuleDescriptors(md0, projectResolver) + def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = + { + val options1 = new ResolveOptions(options0) + var rr = super.resolve(md, options1) + if (!rr.hasError) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) + else { + val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct + val failedPaths = ListMap(rr.getUnresolvedDependencies map { node => + val m = IvyRetrieve.toModuleID(node.getId) + val path = IvyRetrieve.findPath(node, md.getModuleRevisionId) map { x => + IvyRetrieve.toModuleID(x.getId) + } + log.debug("- Unresolved path " + path.toString) + m -> path + }: _*) + val failed = failedPaths.keys.toSeq + Left(new ResolveException(messages, failed, failedPaths)) + } + } + val results = mds map { + case (md, changing) => + cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) { + doWork(md) + } + } + val uReport = mergeResults(md0, results, log) + val cacheManager = getSettings.getResolutionCacheManager + cacheManager.saveResolvedModuleDescriptor(md0) + val prop0 = "" + val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId) + IO.write(ivyPropertiesInCache0, prop0) + uReport + } + def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], log: Logger): Either[ResolveException, UpdateReport] = + if (results exists { _.isLeft }) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) + else Right(mergeReports(md0, results collect { case Right(ur) => ur }, log)) + def mergeErrors(md0: ModuleDescriptor, errors: Vector[ResolveException], log: Logger): ResolveException = + { + val messages = errors flatMap { _.messages } + val failed = errors flatMap { _.failed } + val failedPaths = errors flatMap { + _.failedPaths.toList map { + case (failed, paths) => + if (paths.isEmpty) (failed, paths) + else (failed, List(IvyRetrieve.toModuleID(md0.getResolvedModuleRevisionId)) ::: paths.toList.tail) + } + } + new ResolveException(messages, failed, ListMap(failedPaths: _*)) + } + def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], log: Logger): UpdateReport = + { + val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) + val rootModuleConfigs = md0.getConfigurations.toVector + val stats = new UpdateStats(0L, 0L, 0L, false) + val configReports = rootModuleConfigs map { conf => + val crs = reports flatMap { _.configurations filter { _.configuration == conf.getName } } + mergeConfigurationReports(conf.getName, crs, log) + } + new UpdateReport(cachedDescriptor, configReports, stats, Map.empty) + } + def mergeConfigurationReports(rootModuleConf: String, reports: Vector[ConfigurationReport], log: Logger): ConfigurationReport = + { + // get the details right, and the rest could be derived + val details = mergeOrganizationArtifactReports(rootModuleConf, reports flatMap { _.details }, log) + val modules = details flatMap { + _.modules filter { mr => + !mr.evicted && mr.problem.isEmpty + } + } + val evicted = details flatMap { + _.modules filter { mr => + mr.evicted + } + } map { _.module } + new ConfigurationReport(rootModuleConf, modules, details, evicted) + } + def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], log: Logger): Vector[OrganizationArtifactReport] = + (reports0 groupBy { oar => (oar.organization, oar.name) }).toSeq.toVector flatMap { + case ((org, name), xs) => + if (xs.size < 2) xs + else Vector(new OrganizationArtifactReport(org, name, mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, log))) + } + def mergeModuleReports(rootModuleConf: String, modules: Vector[ModuleReport], log: Logger): Vector[ModuleReport] = + { + val merged = (modules groupBy { m => (m.module.organization, m.module.name, m.module.revision) }).toSeq.toVector flatMap { + case ((org, name, version), xs) => + if (xs.size < 2) xs + else Vector(xs.head.copy(evicted = xs exists { _.evicted }, callers = xs flatMap { _.callers })) + } + val conflicts = merged filter { m => !m.evicted && m.problem.isEmpty } + if (conflicts.size < 2) merged + else resolveConflict(rootModuleConf, conflicts, log) match { + case (survivor, evicted) => + survivor ++ evicted ++ (merged filter { m => m.evicted || m.problem.isDefined }) + } + } + def resolveConflict(rootModuleConf: String, conflicts: Vector[ModuleReport], log: Logger): (Vector[ModuleReport], Vector[ModuleReport]) = + { + import org.apache.ivy.plugins.conflict.{ NoConflictManager, StrictConflictManager, LatestConflictManager } + val head = conflicts.head + val organization = head.module.organization + val name = head.module.name + log.debug(s"- conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")")) + def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) = + conflicts find { m => + m.callers.exists { _.isForceDependency } + } match { + case Some(m) => + (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString) + case None => + val strategy = lcm.getStrategy + val infos = conflicts map { ModuleReportArtifactInfo(_) } + Option(strategy.findLatest(infos.toArray, None.orNull)) match { + case Some(ModuleReportArtifactInfo(m)) => + (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString) + case _ => (conflicts, Vector(), lcm.toString) + } + } + def doResolveConflict: (Vector[ModuleReport], Vector[ModuleReport], String) = + getSettings.getConflictManager(IvyModuleId.newInstance(organization, name)) match { + case ncm: NoConflictManager => (conflicts, Vector(), ncm.toString) + case _: StrictConflictManager => sys.error((s"conflict was found in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")"))) + case lcm: LatestConflictManager => useLatest(lcm) + case conflictManager => sys.error(s"Unsupported conflict manager $conflictManager") + } + if (conflicts.size == 2) { + val (cf0, cf1) = (conflicts(0).module, conflicts(1).module) + val cache = cachedResolutionResolveCache + cache.getOrElseUpdateConflict(cf0, cf1, conflicts) { doResolveConflict } + } else { + val (surviving, evicted, mgr) = doResolveConflict + (surviving, evicted) + } + + } +} + +private[sbt] case class ModuleReportArtifactInfo(moduleReport: ModuleReport) extends IvyArtifactInfo { + override def getLastModified: Long = moduleReport.publicationDate map { _.getTime } getOrElse 0L + override def getRevision: String = moduleReport.module.revision +} diff --git a/ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala deleted file mode 100644 index c853791c8..000000000 --- a/ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala +++ /dev/null @@ -1,120 +0,0 @@ -package sbt -package ivyint - -import java.io.File -import collection.concurrent -import org.apache.ivy.core -import core.resolve._ -import core.module.id.ModuleRevisionId -import core.report.ResolveReport -import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor } -import core.{ IvyPatternHelper, LogOptions } -import org.apache.ivy.util.Message - -private[sbt] object ConsolidatedResolveCache { - def createID(organization: String, name: String, revision: String) = - ModuleRevisionId.newInstance(organization, name, revision) - def sbtOrgTemp = "org.scala-sbt.temp" -} - -private[sbt] class ConsolidatedResolveCache() { - import ConsolidatedResolveCache._ - val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap() - val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap() - val directDependencyCache: concurrent.Map[ModuleDescriptor, Vector[DependencyDescriptor]] = concurrent.TrieMap() - - def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = { - val mrid0 = md0.getModuleRevisionId - val md1 = if (mrid0.getOrganisation == sbtOrgTemp) md0 - else buildConsolidatedModuleDescriptor(md0, prOpt) - val mrid1 = md1.getModuleRevisionId - resolveReportCache.remove(mrid1) - resolvePropertiesCache.remove(mrid1) - } - - def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] = - directDependencyCache.getOrElseUpdate(md0, md0.getDependencies.toVector) - - def buildConsolidatedModuleDescriptor(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): DefaultModuleDescriptor = { - def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] = - prOpt map { - _.getModuleDescriptor(dep.getDependencyRevisionId) match { - case Some(internal) => directDependencies(internal) flatMap expandInternalDeps - case _ => Vector(dep) - } - } getOrElse Vector(dep) - val expanded = directDependencies(md0) flatMap expandInternalDeps - val depStrings = expanded map { dep => - val mrid = dep.getDependencyRevisionId - val confMap = (dep.getModuleConfigurations map { conf => - conf + "->(" + dep.getDependencyConfigurations(conf).mkString(",") + ")" - }) - mrid.toString + ";" + confMap.mkString(";") - } - val depsString = depStrings.distinct.sorted.mkString("\n") - val sha1 = Hash.toHex(Hash(depsString)) - // println("sha1: " + sha1) - val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) - md1 - } -} - -private[sbt] trait ConsolidatedResolveEngine extends ResolveEngine { - import ConsolidatedResolveCache._ - - private[sbt] def consolidatedResolveCache: ConsolidatedResolveCache - private[sbt] def projectResolver: Option[ProjectResolver] - - /** - * Resolve dependencies of a module described by a module descriptor. - */ - override def resolve(md0: ModuleDescriptor, options0: ResolveOptions): ResolveReport = { - val cache = consolidatedResolveCache - val cacheManager = getSettings.getResolutionCacheManager - val md1 = cache.buildConsolidatedModuleDescriptor(md0, projectResolver) - val md1mrid = md1.getModuleRevisionId - - def doWork: (ResolveReport, String) = { - if (options0.getLog != LogOptions.LOG_QUIET) { - Message.info("Consolidating managed dependencies to " + md1mrid.toString + " ...") - } - md1.setLastModified(System.currentTimeMillis) - for { - x <- md0.getConfigurations - } yield md1.addConfiguration(x) - - for { - x <- md0.getDependencies - } yield md1.addDependency(x) - - val options1 = new ResolveOptions(options0) - options1.setOutputReport(false) - val report0 = super.resolve(md1, options1) - val ivyPropertiesInCache1 = cacheManager.getResolvedIvyPropertiesInCache(md1.getResolvedModuleRevisionId) - val prop0 = - if (ivyPropertiesInCache1.exists) IO.read(ivyPropertiesInCache1) - else "" - if (options0.isOutputReport) { - this.outputReport(report0, cacheManager, options0) - } - cache.resolveReportCache(md1mrid) = report0 - cache.resolvePropertiesCache(md1mrid) = prop0 - (report0, prop0) - } - - val (report0, prop0) = (cache.resolveReportCache.get(md1mrid), cache.resolvePropertiesCache.get(md1mrid)) match { - case (Some(report), Some(prop)) => - if (options0.getLog != LogOptions.LOG_QUIET) { - Message.info("Found consolidated dependency " + md1mrid.toString + " ...") - } - (report, prop) - case _ => doWork - } - cacheManager.saveResolvedModuleDescriptor(md0) - if (prop0 != "") { - val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId) - IO.write(ivyPropertiesInCache0, prop0) - } - report0 - } -} diff --git a/ivy/src/test/scala/BaseIvySpecification.scala b/ivy/src/test/scala/BaseIvySpecification.scala index e754903fe..09d7da7a2 100644 --- a/ivy/src/test/scala/BaseIvySpecification.scala +++ b/ivy/src/test/scala/BaseIvySpecification.scala @@ -8,10 +8,12 @@ import cross.CrossVersionUtil trait BaseIvySpecification extends Specification { def currentBase: File = new File(".") def currentTarget: File = currentBase / "target" / "ivyhome" + def currentManaged: File = currentBase / "target" / "lib_managed" + def currentDependency: File = currentBase / "target" / "dependency" def defaultModuleId: ModuleID = ModuleID("com.example", "foo", "0.1.0", Some("compile")) - lazy val ivySbt = new IvySbt(mkIvyConfiguration) - lazy val log = Logger.Null - def module(moduleId: ModuleID, deps: Seq[ModuleID], scalaFullVersion: Option[String]): IvySbt#Module = { + lazy val log = ConsoleLogger() + def module(moduleId: ModuleID, deps: Seq[ModuleID], scalaFullVersion: Option[String], + uo: UpdateOptions = UpdateOptions()): IvySbt#Module = { val ivyScala = scalaFullVersion map { fv => new IvyScala( scalaFullVersion = fv, @@ -28,10 +30,11 @@ trait BaseIvySpecification extends Specification { dependencies = deps, configurations = Seq(Compile, Test, Runtime), ivyScala = ivyScala) + val ivySbt = new IvySbt(mkIvyConfiguration(uo)) new ivySbt.Module(moduleSetting) } - def mkIvyConfiguration: IvyConfiguration = { + def mkIvyConfiguration(uo: UpdateOptions): IvyConfiguration = { val paths = new IvyPaths(currentBase, Some(currentTarget)) val rs = Seq(DefaultMavenRepository) val other = Nil @@ -39,13 +42,20 @@ trait BaseIvySpecification extends Specification { val off = false val check = Nil val resCacheDir = currentTarget / "resolution-cache" - val uo = UpdateOptions() new InlineIvyConfiguration(paths, rs, other, moduleConfs, off, None, check, Some(resCacheDir), uo, log) } - def ivyUpdate(module: IvySbt#Module) = { + def ivyUpdateEither(module: IvySbt#Module): Either[UnresolvedWarning, UpdateReport] = { // IO.delete(currentTarget) - val config = new UpdateConfiguration(None, false, UpdateLogging.Full) - IvyActions.update(module, config, log) + val retrieveConfig = new RetrieveConfiguration(currentManaged, Resolver.defaultRetrievePattern) + val config = new UpdateConfiguration(Some(retrieveConfig), false, UpdateLogging.Full) + IvyActions.updateEither(module, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, Some(currentDependency), log) } + + def ivyUpdate(module: IvySbt#Module) = + ivyUpdateEither(module) match { + case Right(r) => r + case Left(w) => + throw w.resolveException + } } diff --git a/ivy/src/test/scala/CachedResolutionSpec.scala b/ivy/src/test/scala/CachedResolutionSpec.scala new file mode 100644 index 000000000..8ac88740e --- /dev/null +++ b/ivy/src/test/scala/CachedResolutionSpec.scala @@ -0,0 +1,51 @@ +package sbt + +import org.specs2._ + +class CachedResolutionSpec extends BaseIvySpecification { + def is = args(sequential = true) ^ s2""" + + This is a specification to check the cached resolution + + Resolving the same module twice should + work $e1 + + Resolving the unsolvable module should + not work $e2 + """ + + def commonsIo13 = ModuleID("commons-io", "commons-io", "1.3", Some("compile")) + def mavenCayennePlugin302 = ModuleID("org.apache.cayenne.plugins", "maven-cayenne-plugin", "3.0.2", Some("compile")) + + def defaultOptions = EvictionWarningOptions.default + + import ShowLines._ + + def e1 = { + val m = module(ModuleID("com.example", "foo", "0.1.0", Some("compile")), Seq(commonsIo13), Some("2.10.2"), UpdateOptions().withCachedResolution(true)) + val report = ivyUpdate(m) + val report2 = ivyUpdate(m) + println(report) + println(report.configurations.head.modules.head.artifacts) + report.configurations.size must_== 3 + } + + def e2 = { + log.setLevel(Level.Debug) + val m = module(ModuleID("com.example", "foo", "0.2.0", Some("compile")), Seq(mavenCayennePlugin302), Some("2.10.2"), UpdateOptions().withCachedResolution(true)) + ivyUpdateEither(m) match { + case Right(_) => sys.error("this should've failed") + case Left(uw) => + println(uw.lines.mkString("\n")) + } + ivyUpdateEither(m) match { + case Right(_) => sys.error("this should've failed 2") + case Left(uw) => + uw.lines must contain(allOf("\n\tNote: Unresolved dependencies path:", + "\t\tfoundrylogic.vpp:vpp:2.2.1", + "\t\t +- org.apache.cayenne:cayenne-tools:3.0.2", + "\t\t +- org.apache.cayenne.plugins:maven-cayenne-plugin:3.0.2", + "\t\t +- com.example:foo:0.2.0")) + } + } +} diff --git a/main/actions/src/main/scala/sbt/CacheIvy.scala b/main/actions/src/main/scala/sbt/CacheIvy.scala index 9f168d228..17e260450 100644 --- a/main/actions/src/main/scala/sbt/CacheIvy.scala +++ b/main/actions/src/main/scala/sbt/CacheIvy.scala @@ -81,7 +81,8 @@ object CacheIvy { implicit def organizationArtifactReportFormat(implicit sf: Format[String], bf: Format[Boolean], df: Format[Seq[ModuleReport]]): Format[OrganizationArtifactReport] = wrap[OrganizationArtifactReport, (String, String, Seq[ModuleReport])](m => (m.organization, m.name, m.modules), { case (o, n, r) => OrganizationArtifactReport(o, n, r) }) implicit def callerFormat: Format[Caller] = - wrap[Caller, (ModuleID, Seq[String], Map[String, String])](c => (c.caller, c.callerConfigurations, c.callerExtraAttributes), { case (c, cc, ea) => new Caller(c, cc, ea) }) + wrap[Caller, (ModuleID, Seq[String], Map[String, String], Boolean, Boolean, Boolean)](c => (c.caller, c.callerConfigurations, c.callerExtraAttributes, c.isForceDependency, c.isChangingDependency, c.isTransitiveDependency), + { case (c, cc, ea, fd, cd, td) => new Caller(c, cc, ea, fd, cd, td) }) implicit def exclusionRuleFormat(implicit sf: Format[String]): Format[ExclusionRule] = wrap[ExclusionRule, (String, String, String, Seq[String])](e => (e.organization, e.name, e.artifact, e.configurations), { case (o, n, a, cs) => ExclusionRule(o, n, a, cs) }) implicit def crossVersionFormat: Format[CrossVersion] = wrap(crossToInt, crossFromInt) diff --git a/main/src/main/scala/sbt/BuildPaths.scala b/main/src/main/scala/sbt/BuildPaths.scala index 4efe35c1d..76981febe 100644 --- a/main/src/main/scala/sbt/BuildPaths.scala +++ b/main/src/main/scala/sbt/BuildPaths.scala @@ -12,6 +12,7 @@ object BuildPaths { val globalPluginsDirectory = AttributeKey[File]("global-plugins-directory", "The base directory for global sbt plugins.", DSetting) val globalSettingsDirectory = AttributeKey[File]("global-settings-directory", "The base directory for global sbt settings.", DSetting) val stagingDirectory = AttributeKey[File]("staging-directory", "The directory for staging remote projects.", DSetting) + val dependencyBaseDirectory = AttributeKey[File]("dependency-base-directory", "The base directory for caching dependency resolution.", DSetting) import Path._ @@ -39,6 +40,9 @@ object BuildPaths { def getGlobalSettingsDirectory(state: State, globalBase: File): File = fileSetting(globalSettingsDirectory, GlobalSettingsProperty, globalBase)(state) + def getDependencyDirectory(state: State, globalBase: File): File = + fileSetting(dependencyBaseDirectory, DependencyBaseProperty, defaultDependencyBase(globalBase))(state) + private[this] def fileSetting(stateKey: AttributeKey[File], property: String, default: File)(state: State): File = getFileSetting(stateKey, property, default)(state) @@ -56,6 +60,7 @@ object BuildPaths { sbt.cross.CrossVersionUtil.binarySbtVersion(state.configuration.provider.id.version) private[this] def defaultStaging(globalBase: File) = globalBase / "staging" private[this] def defaultGlobalPlugins(globalBase: File) = globalBase / PluginsDirectoryName + private[this] def defaultDependencyBase(globalBase: File) = globalBase / "dependency" def configurationSources(base: File): Seq[File] = (base * (GlobFilter("*.sbt") - ".sbt")).get def pluginDirectory(definitionBase: File) = definitionBase / PluginsDirectoryName @@ -77,6 +82,7 @@ object BuildPaths { final val StagingProperty = "sbt.global.staging" final val GlobalPluginsProperty = "sbt.global.plugins" final val GlobalSettingsProperty = "sbt.global.settings" + final val DependencyBaseProperty = "sbt.dependency.base" def crossPath(base: File, instance: xsbti.compile.ScalaInstance): File = base / ("scala_" + instance.version) diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 58ddd7efa..28fb7be54 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -222,7 +222,7 @@ object Defaults extends BuildCommon { crossTarget := makeCrossTarget(target.value, scalaBinaryVersion.value, sbtBinaryVersion.value, sbtPlugin.value, crossPaths.value), clean := { val _ = clean.value - IvyActions.cleanConsolidatedResolutionCache(ivyModule.value, streams.value.log) + IvyActions.cleanCachedResolutionCache(ivyModule.value, streams.value.log) } ) // must be a val: duplication detected by object identity @@ -1255,6 +1255,8 @@ object Classpaths { case (Some(res), _, Some(decl)) if res == decl => jars case _ => Nil } + def intToByteArray(x: Int): Array[Byte] = + Array((x >>> 24).toByte, (x >> 16 & 0xff).toByte, (x >> 8 & 0xff).toByte, (x & 0xff).toByte) val subScalaJars: String => Seq[File] = Defaults.unmanagedScalaInstanceOnly.value match { case Some(si) => subUnmanaged(si.version, si.jars) case None => sv => if (scalaProvider.version == sv) scalaProvider.jars else Nil @@ -1262,25 +1264,28 @@ object Classpaths { val transform: UpdateReport => UpdateReport = r => substituteScalaFiles(scalaOrganization.value, r)(subScalaJars) val uwConfig = (unresolvedWarningConfiguration in update).value val show = Reference.display(thisProjectRef.value) + val st = state.value + val logicalClock = LogicalClock(Hash.toHex(intToByteArray(st.hashCode))) + val depDir = BuildPaths.getDependencyDirectory(st, BuildPaths.getGlobalBase(st)) cachedUpdate(s.cacheDirectory / updateCacheName.value, show, ivyModule.value, updateConfiguration.value, transform, skip = (skip in update).value, force = isRoot, depsUpdated = depsUpdated, - uwConfig = uwConfig, log = s.log) + uwConfig = uwConfig, logicalClock = logicalClock, depDir = Some(depDir), log = s.log) } @deprecated("Use cachedUpdate with the variant that takes unresolvedHandler instead.", "0.13.6") def cachedUpdate(cacheFile: File, label: String, module: IvySbt#Module, config: UpdateConfiguration, transform: UpdateReport => UpdateReport, skip: Boolean, force: Boolean, depsUpdated: Boolean, log: Logger): UpdateReport = cachedUpdate(cacheFile, label, module, config, transform, skip, force, depsUpdated, - UnresolvedWarningConfiguration(), log) - def cachedUpdate(cacheFile: File, label: String, module: IvySbt#Module, config: UpdateConfiguration, + UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) + private[sbt] def cachedUpdate(cacheFile: File, label: String, module: IvySbt#Module, config: UpdateConfiguration, transform: UpdateReport => UpdateReport, skip: Boolean, force: Boolean, depsUpdated: Boolean, - uwConfig: UnresolvedWarningConfiguration, log: Logger): UpdateReport = + uwConfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): UpdateReport = { implicit val updateCache = updateIC type In = IvyConfiguration :+: ModuleSettings :+: UpdateConfiguration :+: HNil def work = (_: In) match { case conf :+: settings :+: config :+: HNil => log.info("Updating " + label + "...") - val r = IvyActions.updateEither(module, config, uwConfig, log) match { + val r = IvyActions.updateEither(module, config, uwConfig, logicalClock, depDir, log) match { case Right(ur) => ur case Left(uw) => import ShowLines._ diff --git a/project/Sbt.scala b/project/Sbt.scala index bfe0a386f..0c23ce6d2 100644 --- a/project/Sbt.scala +++ b/project/Sbt.scala @@ -126,7 +126,7 @@ object Sbt extends Build { /* **** Intermediate-level Modules **** */ // Apache Ivy integration - lazy val ivySub = baseProject(file("ivy"), "Ivy") dependsOn (interfaceSub, launchInterfaceSub, crossSub, logSub % "compile;test->test", ioSub % "compile;test->test", launchSub % "test->test", collectionSub) settings (ivy, jsch, testExclusive) + lazy val ivySub = baseProject(file("ivy"), "Ivy") dependsOn (interfaceSub, launchInterfaceSub, crossSub, logSub % "compile;test->test", ioSub % "compile;test->test", launchSub % "test->test", collectionSub) settings (ivy, jsch, testExclusive, json4sNative, jawnParser, jawnJson4s) // Runner for uniform test interface lazy val testingSub = baseProject(file("testing"), "Testing") dependsOn (ioSub, classpathSub, logSub, launchInterfaceSub, testAgentSub) settings (testInterface) // Testing agent for running tests in a separate process. diff --git a/project/Util.scala b/project/Util.scala index 1aa735f24..dec9ffd37 100644 --- a/project/Util.scala +++ b/project/Util.scala @@ -178,6 +178,9 @@ object Common { lazy val httpclient = lib("commons-httpclient" % "commons-httpclient" % "3.1") lazy val jsch = lib("com.jcraft" % "jsch" % "0.1.46" intransitive ()) lazy val sbinary = libraryDependencies += "org.scala-tools.sbinary" %% "sbinary" % "0.4.2" + lazy val json4sNative = lib("org.json4s" %% "json4s-native" % "3.2.10") + lazy val jawnParser = lib("org.spire-math" %% "jawn-parser" % "0.6.0") + lazy val jawnJson4s = lib("org.spire-math" %% "json4s-support" % "0.6.0") lazy val scalaCompiler = libraryDependencies <+= scalaVersion(sv => "org.scala-lang" % "scala-compiler" % sv) lazy val testInterface = lib("org.scala-sbt" % "test-interface" % "1.0") private def scala211Module(name: String, moduleVersion: String) = diff --git a/sbt/src/sbt-test/dependency-management/consolidated-resolution/multi.sbt b/sbt/src/sbt-test/dependency-management/consolidated-resolution/multi.sbt index 6718a5d94..341ab792f 100644 --- a/sbt/src/sbt-test/dependency-management/consolidated-resolution/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/consolidated-resolution/multi.sbt @@ -6,10 +6,13 @@ def commonSettings: Seq[Def.Setting[_]] = libraryDependencies := Seq( "net.sf.json-lib" % "json-lib" % "2.4" classifier "jdk15" intransitive(), "com.typesafe.akka" %% "akka-remote" % "2.3.4" exclude("com.typesafe.akka", "akka-actor_2.10"), - "commons-io" % "commons-io" % "1.3" + "net.databinder" %% "unfiltered-uploads" % "0.8.0", + "commons-io" % "commons-io" % "1.3", + "com.typesafe" % "config" % "0.4.9-SNAPSHOT" ), - dependencyOverrides += "commons-io" % "commons-io" % "1.4", - scalaVersion := "2.10.4" + // dependencyOverrides += "commons-io" % "commons-io" % "1.4", + scalaVersion := "2.10.4", + resolvers += Resolver.sonatypeRepo("snapshots") ) def consolidatedResolutionSettings: Seq[Def.Setting[_]] = @@ -27,9 +30,10 @@ lazy val b = project. settings(commonSettings: _*) lazy val c = project. + dependsOn(a). settings(consolidatedResolutionSettings: _*). settings( - libraryDependencies := Seq(organization.value %% "a" % version.value) + // libraryDependencies := Seq(organization.value %% "a" % version.value) ) lazy val root = (project in file(".")). @@ -37,9 +41,13 @@ lazy val root = (project in file(".")). organization in ThisBuild := "org.example", version in ThisBuild := "1.0", check := { - val acp = (externalDependencyClasspath in Compile in a).value - val bcp = (externalDependencyClasspath in Compile in b).value - if (acp == bcp) () - else sys.error("Different classpaths are found:\n" + acp.toString + "\n" + bcp.toString) + val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName} + val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName} + val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} filterNot {_.data.getName == "demo_2.10.jar"} + if (acp == bcp && acp == ccp) () + else sys.error("Different classpaths are found:" + + "\n - a (consolidated) " + acp.toString + + "\n - b (plain) " + bcp.toString + + "\n - c (inter-project) " + ccp.toString) } ) diff --git a/sbt/src/sbt-test/dependency-management/consolidated-resolution/test b/sbt/src/sbt-test/dependency-management/consolidated-resolution/test index 7394143cd..7e1f55e8e 100644 --- a/sbt/src/sbt-test/dependency-management/consolidated-resolution/test +++ b/sbt/src/sbt-test/dependency-management/consolidated-resolution/test @@ -2,4 +2,4 @@ > check -> c/run +# > c/run From 1428a5685d709cd5741e67f8f49143aa5922ea69 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 30 Sep 2014 17:30:51 -0400 Subject: [PATCH 2/6] spawn new instance of Ivy during minigraph resolution --- ivy/src/main/scala/sbt/Ivy.scala | 1 + .../scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index db8d8ed1e..2ea187a23 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -98,6 +98,7 @@ final class IvySbt(val configuration: IvyConfiguration) { setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine) with CachedResolutionResolveEngine { val cachedResolutionResolveCache = IvySbt.cachedResolutionResolveCache val projectResolver = prOpt + def makeInstance = mkIvy }) } else setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine)) super.bind() diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 9db9343bb..466f38ca3 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -6,6 +6,7 @@ import java.net.URL import java.io.File import collection.concurrent import collection.immutable.ListMap +import org.apache.ivy.Ivy import org.apache.ivy.core import core.resolve._ import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } @@ -260,6 +261,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { private[sbt] def cachedResolutionResolveCache: CachedResolutionResolveCache private[sbt] def projectResolver: Option[ProjectResolver] + private[sbt] def makeInstance: Ivy // Return sbt's UpdateReport. def customResolve(md0: ModuleDescriptor, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { @@ -271,7 +273,8 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = { val options1 = new ResolveOptions(options0) - var rr = super.resolve(md, options1) + val i = makeInstance + var rr = i.resolve(md, options1) if (!rr.hasError) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) else { val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct From 6c3b95b23d961cab30c4547d72bbbddaf9f5f3f6 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 30 Sep 2014 17:44:56 -0400 Subject: [PATCH 3/6] split up json persistence into JsonUtil.scala --- ivy/src/main/scala/sbt/JsonUtil.scala | 110 ++++++++++++++++++ .../CachedResolutionResolveEngine.scala | 110 +----------------- 2 files changed, 116 insertions(+), 104 deletions(-) create mode 100644 ivy/src/main/scala/sbt/JsonUtil.scala diff --git a/ivy/src/main/scala/sbt/JsonUtil.scala b/ivy/src/main/scala/sbt/JsonUtil.scala new file mode 100644 index 000000000..ba843aec1 --- /dev/null +++ b/ivy/src/main/scala/sbt/JsonUtil.scala @@ -0,0 +1,110 @@ +package sbt + +import java.io.File +import java.net.URL +import org.json4s._ +import org.apache.ivy.core +import core.module.descriptor.ModuleDescriptor + +private[sbt] object JsonUtil { + def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = + { + import org.json4s._ + implicit val formats = native.Serialization.formats(NoTypeHints) + + new ConfigurationSerializer + + new ArtifactSerializer + + new FileSerializer + try { + val json = jawn.support.json4s.Parser.parseFromFile(path) + fromLite(json.get.extract[UpdateReportLite], cachedDescriptor) + } catch { + case e: Throwable => + log.error("Unable to parse mini graph: " + path.toString) + throw e + } + } + def writeUpdateReport(ur: UpdateReport, graphPath: File): Unit = + { + implicit val formats = native.Serialization.formats(NoTypeHints) + + new ConfigurationSerializer + + new ArtifactSerializer + + new FileSerializer + import native.Serialization.write + val str = write(toLite(ur)) + IO.write(graphPath, str, IO.utf8) + } + def toLite(ur: UpdateReport): UpdateReportLite = + UpdateReportLite(ur.configurations map { cr => + ConfigurationReportLite(cr.configuration, cr.details) + }) + def fromLite(lite: UpdateReportLite, cachedDescriptor: File): UpdateReport = + { + val stats = new UpdateStats(0L, 0L, 0L, false) + val configReports = lite.configurations map { cr => + val details = cr.details + val modules = details flatMap { + _.modules filter { mr => + !mr.evicted && mr.problem.isEmpty + } + } + val evicted = details flatMap { + _.modules filter { mr => + mr.evicted + } + } map { _.module } + new ConfigurationReport(cr.configuration, modules, details, evicted) + } + new UpdateReport(cachedDescriptor, configReports, stats) + } +} + +private[sbt] case class UpdateReportLite(configurations: Seq[ConfigurationReportLite]) +private[sbt] case class ConfigurationReportLite(configuration: String, details: Seq[OrganizationArtifactReport]) + +private[sbt] class FileSerializer extends CustomSerializer[File](format => ( + { + case JString(s) => new File(s) + }, + { + case x: File => JString(x.toString) + } +)) + +private[sbt] class ConfigurationSerializer extends CustomSerializer[Configuration](format => ( + { + case JString(s) => new Configuration(s) + }, + { + case x: Configuration => JString(x.name) + } +)) + +private[sbt] class ArtifactSerializer extends CustomSerializer[Artifact](format => ( + { + case json: JValue => + implicit val fmt = format + Artifact( + (json \ "name").extract[String], + (json \ "type").extract[String], + (json \ "extension").extract[String], + (json \ "classifier").extract[Option[String]], + (json \ "configurations").extract[List[Configuration]], + (json \ "url").extract[Option[URL]], + (json \ "extraAttributes").extract[Map[String, String]] + ) + }, + { + case x: Artifact => + import DefaultJsonFormats.{ OptionWriter, StringWriter, mapWriter } + val optStr = implicitly[Writer[Option[String]]] + val mw = implicitly[Writer[Map[String, String]]] + JObject(JField("name", JString(x.name)) :: + JField("type", JString(x.`type`)) :: + JField("extension", JString(x.extension)) :: + JField("classifier", optStr.write(x.classifier)) :: + JField("configurations", JArray(x.configurations.toList map { x => JString(x.name) })) :: + JField("url", optStr.write(x.url map { _.toString })) :: + JField("extraAttributes", mw.write(x.extraAttributes)) :: + Nil) + } +)) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 466f38ca3..e2c71a877 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -15,7 +15,6 @@ import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, Depen import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.Message import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } -import org.json4s._ private[sbt] object CachedResolutionResolveCache { def createID(organization: String, name: String, revision: String) = @@ -39,7 +38,6 @@ class CachedResolutionResolveCache() { else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 } mds foreach { md => updateReportCache.remove(md.getModuleRevisionId) - // todo: backport this directDependencyCache.remove(md.getModuleRevisionId) } } @@ -98,7 +96,7 @@ class CachedResolutionResolveCache() { else if (dynamicGraphPath.exists) Some(dynamicGraphPath) else None) map { path => log.debug(s"parsing ${path.getAbsolutePath.toString}") - val ur = parseUpdateReport(md, path, cachedDescriptor, log) + val ur = JsonUtil.parseUpdateReport(md, path, cachedDescriptor, log) updateReportCache(md.getModuleRevisionId) = Right(ur) Right(ur) } @@ -115,9 +113,10 @@ class CachedResolutionResolveCache() { } }) IO.createDirectory(miniGraphPath) - writeUpdateReport(ur, - if (changing) dynamicGraphPath - else staticGraphPath) + val gp = if (changing) dynamicGraphPath + else staticGraphPath + log.debug(s"saving minigraph to $gp") + JsonUtil.writeUpdateReport(ur, gp) // don't cache dynamic graphs in memory. if (!changing) { updateReportCache(md.getModuleRevisionId) = Right(ur) @@ -131,56 +130,7 @@ class CachedResolutionResolveCache() { } } } - def parseUpdateReport(md: ModuleDescriptor, path: File, cachedDescriptor: File, log: Logger): UpdateReport = - { - import org.json4s._ - implicit val formats = native.Serialization.formats(NoTypeHints) + - new ConfigurationSerializer + - new ArtifactSerializer + - new FileSerializer - try { - val json = jawn.support.json4s.Parser.parseFromFile(path) - fromLite(json.get.extract[UpdateReportLite], cachedDescriptor) - } catch { - case e: Throwable => - log.error("Unable to parse mini graph: " + path.toString) - throw e - } - } - def writeUpdateReport(ur: UpdateReport, graphPath: File): Unit = - { - implicit val formats = native.Serialization.formats(NoTypeHints) + - new ConfigurationSerializer + - new ArtifactSerializer + - new FileSerializer - import native.Serialization.write - println(graphPath.toString) - val str = write(toLite(ur)) - IO.write(graphPath, str, IO.utf8) - } - def toLite(ur: UpdateReport): UpdateReportLite = - UpdateReportLite(ur.configurations map { cr => - ConfigurationReportLite(cr.configuration, cr.details) - }) - def fromLite(lite: UpdateReportLite, cachedDescriptor: File): UpdateReport = - { - val stats = new UpdateStats(0L, 0L, 0L, false) - val configReports = lite.configurations map { cr => - val details = cr.details - val modules = details flatMap { - _.modules filter { mr => - !mr.evicted && mr.problem.isEmpty - } - } - val evicted = details flatMap { - _.modules filter { mr => - mr.evicted - } - } map { _.module } - new ConfigurationReport(cr.configuration, modules, details, evicted) - } - new UpdateReport(cachedDescriptor, configReports, stats) - } + def getOrElseUpdateConflict(cf0: ModuleID, cf1: ModuleID, conflicts: Vector[ModuleReport])(f: => (Vector[ModuleReport], Vector[ModuleReport], String)): (Vector[ModuleReport], Vector[ModuleReport]) = { def reconstructReports(surviving: Vector[ModuleID], evicted: Vector[ModuleID], mgr: String): (Vector[ModuleReport], Vector[ModuleReport]) = { @@ -202,60 +152,12 @@ class CachedResolutionResolveCache() { } } } - class FileSerializer extends CustomSerializer[File](format => ( - { - case JString(s) => new File(s) - }, - { - case x: File => JString(x.toString) - } - )) - class ConfigurationSerializer extends CustomSerializer[Configuration](format => ( - { - case JString(s) => new Configuration(s) - }, - { - case x: Configuration => JString(x.name) - } - )) - class ArtifactSerializer extends CustomSerializer[Artifact](format => ( - { - case json: JValue => - implicit val fmt = format - Artifact( - (json \ "name").extract[String], - (json \ "type").extract[String], - (json \ "extension").extract[String], - (json \ "classifier").extract[Option[String]], - (json \ "configurations").extract[List[Configuration]], - (json \ "url").extract[Option[URL]], - (json \ "extraAttributes").extract[Map[String, String]] - ) - }, - { - case x: Artifact => - import DefaultJsonFormats.{ OptionWriter, StringWriter, mapWriter } - val optStr = implicitly[Writer[Option[String]]] - val mw = implicitly[Writer[Map[String, String]]] - JObject(JField("name", JString(x.name)) :: - JField("type", JString(x.`type`)) :: - JField("extension", JString(x.extension)) :: - JField("classifier", optStr.write(x.classifier)) :: - JField("configurations", JArray(x.configurations.toList map { x => JString(x.name) })) :: - JField("url", optStr.write(x.url map { _.toString })) :: - JField("extraAttributes", mw.write(x.extraAttributes)) :: - Nil) - } - )) } private[sbt] trait ArtificialModuleDescriptor { self: DefaultModuleDescriptor => def targetModuleRevisionId: ModuleRevisionId } -private[sbt] case class UpdateReportLite(configurations: Seq[ConfigurationReportLite]) -private[sbt] case class ConfigurationReportLite(configuration: String, details: Seq[OrganizationArtifactReport]) - private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { import CachedResolutionResolveCache._ From 02249c13822395b126abcbacb1659819dc14725b Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 30 Sep 2014 17:48:17 -0400 Subject: [PATCH 4/6] update deprecation message --- ivy/src/main/scala/sbt/IvyActions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/IvyActions.scala b/ivy/src/main/scala/sbt/IvyActions.scala index 0d12d5b06..f5e57336d 100644 --- a/ivy/src/main/scala/sbt/IvyActions.scala +++ b/ivy/src/main/scala/sbt/IvyActions.scala @@ -140,7 +140,7 @@ object IvyActions { * Resolves and retrieves dependencies. 'ivyConfig' is used to produce an Ivy file and configuration. * 'updateConfig' configures the actual resolution and retrieval process. */ - @deprecated("Use updateEither instead.", "0.13.6") + @deprecated("This is no longer public.", "0.13.6") def update(module: IvySbt#Module, configuration: UpdateConfiguration, log: Logger): UpdateReport = updateEither(module, configuration, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) match { case Right(r) => r From 40b91b5a717887d292c3fd68ed132f518f046c4a Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 30 Sep 2014 18:19:46 -0400 Subject: [PATCH 5/6] some more minor changes --- ivy/src/main/scala/sbt/Ivy.scala | 2 ++ .../scala/sbt/ivyint/CachedResolutionResolveEngine.scala | 6 ++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 2ea187a23..4bf6ffad6 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -273,6 +273,8 @@ private[sbt] object IvySbt { val mainChain = makeChain("Default", "sbt-chain", resolvers) settings.setDefaultResolver(mainChain.getName) } + private[sbt] def isChanging(dd: DependencyDescriptor): Boolean = + dd.isChanging || isChanging(dd.getDependencyRevisionId) private[sbt] def isChanging(module: ModuleID): Boolean = module.revision endsWith "-SNAPSHOT" private[sbt] def isChanging(mrid: ModuleRevisionId): Boolean = diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index e2c71a877..4b6cd3e9e 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -22,7 +22,7 @@ private[sbt] object CachedResolutionResolveCache { def sbtOrgTemp = "org.scala-sbt.temp" } -class CachedResolutionResolveCache() { +private[sbt] class CachedResolutionResolveCache() { import CachedResolutionResolveCache._ val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap() val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap() @@ -57,8 +57,6 @@ class CachedResolutionResolveCache() { val rootModuleConfigs = md0.getConfigurations.toVector expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, prOpt) } } - def isDependencyChanging(dd: DependencyDescriptor): Boolean = - dd.isChanging || IvySbt.isChanging(dd.getDependencyRevisionId) def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) = { val mrid = dd.getDependencyRevisionId @@ -74,7 +72,7 @@ class CachedResolutionResolveCache() { conf <- rootModuleConfigs } yield md1.addConfiguration(conf) md1.addDependency(dd) - (md1, isDependencyChanging(dd)) + (md1, IvySbt.isChanging(dd)) } def getOrElseUpdateMiniGraph(md: ModuleDescriptor, changing0: Boolean, logicalClock: LogicalClock, miniGraphPath: File, cachedDescriptor: File, log: Logger)(f: => Either[ResolveException, UpdateReport]): Either[ResolveException, UpdateReport] = { From 4cd895e92a00ebdfe01210f6197063d4ef690257 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 30 Sep 2014 21:57:18 -0400 Subject: [PATCH 6/6] notes --- notes/0.13.7/cached-resolution.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 notes/0.13.7/cached-resolution.md diff --git a/notes/0.13.7/cached-resolution.md b/notes/0.13.7/cached-resolution.md new file mode 100644 index 000000000..8c7339161 --- /dev/null +++ b/notes/0.13.7/cached-resolution.md @@ -0,0 +1,16 @@ + [@eed3si9n]: https://github.com/eed3si9n + [1631]: https://github.com/sbt/sbt/pull/1631 + +### cached resolution (minigraph caching) + +sbt 0.13.7 adds a new update option called *cached resolution*, which replaces consolidated resolution: + + updateOptions := updateOptions.value.withCachedResolution(true) + +Unlike consolidated resolution, which only consolidated subprojects with identical dependency graph, cached resolution create an artificial graph for each direct dependency (minigraph) for all subprojects, resolves them independently, saves them into json file, and stiches the minigraphs together. + +Once the minigraphs are resolved and saved as files, dependency resolution turns into a matter of loading json file from the second run onwards, which should complete in a matter of seconds even for large projects. Also, because the files are saved under a global `~/.sbt/0.13/dependency` (or what's specified by `sbt.dependency.base` flag), the resolution result is shared across all builds. + +Breaking graphs into minigraphs allows partial resolution results to be shared, which scales better for subprojects with similar but slightly different dependencies, and also for making small changes to the dependencies graph over time. + +[#1631][1631] by [@eed3si9n][@eed3si9n].