diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 7f54fb4b4..78c8e3b99 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -4,6 +4,7 @@ package sbt import Resolver.PluginPattern +import ivyint.{ ConsolidatedResolveEngine, ConsolidatedResolveCache } import java.io.File import java.net.URI @@ -14,12 +15,14 @@ import CS.singleton import org.apache.ivy.Ivy import org.apache.ivy.core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.core.cache.{ CacheMetadataOptions, DefaultRepositoryCacheManager, ModuleDescriptorWriter } +import org.apache.ivy.core.event.EventManager import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, MDArtifact } import org.apache.ivy.core.module.descriptor.{ DefaultDependencyDescriptor, DefaultModuleDescriptor, DependencyDescriptor, ModuleDescriptor, License } import org.apache.ivy.core.module.descriptor.{ OverrideDependencyDescriptorMediator } import org.apache.ivy.core.module.id.{ ArtifactId, ModuleId, ModuleRevisionId } -import org.apache.ivy.core.resolve.{ IvyNode, ResolveData, ResolvedModuleRevision } +import org.apache.ivy.core.resolve.{ IvyNode, ResolveData, ResolvedModuleRevision, ResolveEngine } import org.apache.ivy.core.settings.IvySettings +import org.apache.ivy.core.sort.SortEngine import org.apache.ivy.plugins.latest.LatestRevisionStrategy import org.apache.ivy.plugins.matcher.PatternMatcher import org.apache.ivy.plugins.parser.m2.PomModuleDescriptorParser @@ -28,6 +31,7 @@ import org.apache.ivy.util.{ Message, MessageLogger } import org.apache.ivy.util.extendable.ExtendableItem import scala.xml.{ NodeSeq, Text } +import scala.collection.mutable final class IvySbt(val configuration: IvyConfiguration) { import configuration.baseDirectory @@ -76,7 +80,23 @@ final class IvySbt(val configuration: IvyConfiguration) { } private lazy val ivy: Ivy = { - val i = new Ivy() { private val loggerEngine = new SbtMessageLoggerEngine; override def getLoggerEngine = loggerEngine } + val i = new Ivy() { + private val loggerEngine = new SbtMessageLoggerEngine + override def getLoggerEngine = loggerEngine + override def bind(): Unit = { + val prOpt = Option(getSettings.getResolver(ProjectResolver.InterProject)) map { case pr: ProjectResolver => pr } + // We inject the deps we need before we can hook our resolve engine. + setSortEngine(new SortEngine(getSettings)) + setEventManager(new EventManager()) + if (configuration.updateOptions.consolidatedResolution) { + setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine) with ConsolidatedResolveEngine { + val consolidatedResolveCache = IvySbt.consolidatedResolveCache + val projectResolver = prOpt + }) + } else setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine)) + super.bind() + } + } i.setSettings(settings) i.bind() i.getLoggerEngine.pushLogger(new IvyLoggerInterface(configuration.log)) @@ -103,6 +123,18 @@ final class IvySbt(val configuration: IvyConfiguration) { } } + /** + * Cleans consolidated resolution cache. + * @param md - module descriptor of the original Ivy graph. + */ + private[sbt] def cleanConsolidatedResolutionCache(md: ModuleDescriptor, log: Logger): Unit = + withIvy(log) { i => + val prOpt = Option(i.getSettings.getResolver(ProjectResolver.InterProject)) map { case pr: ProjectResolver => pr } + if (configuration.updateOptions.consolidatedResolution) { + IvySbt.consolidatedResolveCache.clean(md, prOpt) + } + } + final class Module(rawModuleSettings: ModuleSettings) { val moduleSettings: ModuleSettings = IvySbt.substituteCross(rawModuleSettings) def owner = IvySbt.this @@ -204,6 +236,7 @@ private object IvySbt { val DefaultIvyFilename = "ivy.xml" val DefaultMavenFilename = "pom.xml" val DefaultChecksums = Seq("sha1", "md5") + private[sbt] val consolidatedResolveCache: ConsolidatedResolveCache = new ConsolidatedResolveCache() def defaultIvyFile(project: File) = new File(project, DefaultIvyFilename) def defaultIvyConfiguration(project: File) = new File(project, DefaultIvyConfigFilename) diff --git a/ivy/src/main/scala/sbt/IvyActions.scala b/ivy/src/main/scala/sbt/IvyActions.scala index bdbf5fe1c..4960c9a2d 100644 --- a/ivy/src/main/scala/sbt/IvyActions.scala +++ b/ivy/src/main/scala/sbt/IvyActions.scala @@ -59,6 +59,15 @@ object IvyActions { iv.getSettings.getRepositoryCacheManagers.foreach(_.clean()) } + /** + * Cleans the consolidated resolution cache, if any. + * This is called by clean. + */ + private[sbt] def cleanConsolidatedResolutionCache(module: IvySbt#Module, log: Logger): Unit = + module.withModule(log) { (ivy, md, default) => + module.owner.cleanConsolidatedResolutionCache(md, log) + } + /** Creates a Maven pom from the given Ivy configuration*/ def makePom(module: IvySbt#Module, configuration: MakePomConfiguration, log: Logger) { import configuration.{ allRepositories, moduleInfo, configurations, extra, file, filterRepositories, process, includeTypes } diff --git a/ivy/src/main/scala/sbt/IvyConfigurations.scala b/ivy/src/main/scala/sbt/IvyConfigurations.scala index 93025e2d7..b3e1dba1b 100644 --- a/ivy/src/main/scala/sbt/IvyConfigurations.scala +++ b/ivy/src/main/scala/sbt/IvyConfigurations.scala @@ -16,27 +16,42 @@ sealed trait IvyConfiguration { def baseDirectory: File def log: Logger def withBase(newBaseDirectory: File): This + def updateOptions: UpdateOptions } final class InlineIvyConfiguration(val paths: IvyPaths, val resolvers: Seq[Resolver], val otherResolvers: Seq[Resolver], val moduleConfigurations: Seq[ModuleConfiguration], val localOnly: Boolean, val lock: Option[xsbti.GlobalLock], - val checksums: Seq[String], val resolutionCacheDir: Option[File], val log: Logger) extends IvyConfiguration { - @deprecated("Use the variant that accepts the resolution cache location.", "0.13.0") + val checksums: Seq[String], val resolutionCacheDir: Option[File], val updateOptions: UpdateOptions, + val log: Logger) extends IvyConfiguration { + @deprecated("Use the variant that accepts resolutionCacheDir and updateOptions.", "0.13.0") def this(paths: IvyPaths, resolvers: Seq[Resolver], otherResolvers: Seq[Resolver], moduleConfigurations: Seq[ModuleConfiguration], localOnly: Boolean, lock: Option[xsbti.GlobalLock], checksums: Seq[String], log: Logger) = - this(paths, resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, None, log) + this(paths, resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, None, UpdateOptions(), log) + + @deprecated("Use the variant that accepts updateOptions.", "0.13.6") + def this(paths: IvyPaths, resolvers: Seq[Resolver], otherResolvers: Seq[Resolver], + moduleConfigurations: Seq[ModuleConfiguration], localOnly: Boolean, lock: Option[xsbti.GlobalLock], + checksums: Seq[String], resolutionCacheDir: Option[File], log: Logger) = + this(paths, resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, resolutionCacheDir, UpdateOptions(), log) type This = InlineIvyConfiguration def baseDirectory = paths.baseDirectory - def withBase(newBase: File) = new InlineIvyConfiguration(paths.withBase(newBase), resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, resolutionCacheDir, log) - def changeResolvers(newResolvers: Seq[Resolver]) = new InlineIvyConfiguration(paths, newResolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, resolutionCacheDir, log) + def withBase(newBase: File) = new InlineIvyConfiguration(paths.withBase(newBase), resolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, + resolutionCacheDir, updateOptions, log) + def changeResolvers(newResolvers: Seq[Resolver]) = new InlineIvyConfiguration(paths, newResolvers, otherResolvers, moduleConfigurations, localOnly, lock, checksums, + resolutionCacheDir, updateOptions, log) } -final class ExternalIvyConfiguration(val baseDirectory: File, val uri: URI, val lock: Option[xsbti.GlobalLock], val extraResolvers: Seq[Resolver], val log: Logger) extends IvyConfiguration { +final class ExternalIvyConfiguration(val baseDirectory: File, val uri: URI, val lock: Option[xsbti.GlobalLock], + val extraResolvers: Seq[Resolver], val updateOptions: UpdateOptions, val log: Logger) extends IvyConfiguration { + @deprecated("Use the variant that accepts updateOptions.", "0.13.6") + def this(baseDirectory: File, uri: URI, lock: Option[xsbti.GlobalLock], extraResolvers: Seq[Resolver], log: Logger) = + this(baseDirectory, uri, lock, extraResolvers, UpdateOptions(), log) + type This = ExternalIvyConfiguration def withBase(newBase: File) = new ExternalIvyConfiguration(newBase, uri, lock, extraResolvers, log) } object ExternalIvyConfiguration { - def apply(baseDirectory: File, file: File, lock: Option[xsbti.GlobalLock], log: Logger) = new ExternalIvyConfiguration(baseDirectory, file.toURI, lock, Nil, log) + def apply(baseDirectory: File, file: File, lock: Option[xsbti.GlobalLock], log: Logger) = new ExternalIvyConfiguration(baseDirectory, file.toURI, lock, Nil, UpdateOptions(), log) } object IvyConfiguration { diff --git a/ivy/src/main/scala/sbt/ProjectResolver.scala b/ivy/src/main/scala/sbt/ProjectResolver.scala index 92fa48bfe..a51e0abb8 100644 --- a/ivy/src/main/scala/sbt/ProjectResolver.scala +++ b/ivy/src/main/scala/sbt/ProjectResolver.scala @@ -34,6 +34,8 @@ class ProjectResolver(name: String, map: Map[ModuleRevisionId, ModuleDescriptor] map get revisionId map constructResult } + private[sbt] def getModuleDescriptor(revisionId: ModuleRevisionId): Option[ModuleDescriptor] = map.get(revisionId) + def report(revisionId: ModuleRevisionId): MetadataArtifactDownloadReport = { val artifact = DefaultArtifact.newIvyArtifact(revisionId, new Date) @@ -87,3 +89,7 @@ class ProjectResolver(name: String, map: Map[ModuleRevisionId, ModuleDescriptor] def setSettings(settings: ResolverSettings) { this.settings = Some(settings) } def getRepositoryCacheManager = settings match { case Some(s) => s.getDefaultRepositoryCacheManager; case None => sys.error("No settings defined for ProjectResolver") } } + +object ProjectResolver { + private[sbt] val InterProject = "inter-project" +} diff --git a/ivy/src/main/scala/sbt/UpdateOptions.scala b/ivy/src/main/scala/sbt/UpdateOptions.scala new file mode 100644 index 000000000..acea391f7 --- /dev/null +++ b/ivy/src/main/scala/sbt/UpdateOptions.scala @@ -0,0 +1,27 @@ +package sbt + +import java.io.File + +/** + * Represents configurable options for update task. + * While UpdateConfiguration is passed into update at runtime, + * UpdateOption is intended to be used while setting up the Ivy object. + * + * See also UpdateConfiguration in IvyActions.scala. + */ +final class UpdateOptions( + /** If set to true, use consolidated resolution. */ + val consolidatedResolution: Boolean) { + + def withConsolidatedResolution(consolidatedResolution: Boolean): UpdateOptions = + copy(consolidatedResolution = consolidatedResolution) + + private[sbt] def copy( + consolidatedResolution: Boolean = this.consolidatedResolution): UpdateOptions = + new UpdateOptions(consolidatedResolution) +} + +object UpdateOptions { + def apply(): UpdateOptions = + new UpdateOptions(false) +} diff --git a/ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala new file mode 100644 index 000000000..c853791c8 --- /dev/null +++ b/ivy/src/main/scala/sbt/ivyint/ConsolidatedResolveEngine.scala @@ -0,0 +1,120 @@ +package sbt +package ivyint + +import java.io.File +import collection.concurrent +import org.apache.ivy.core +import core.resolve._ +import core.module.id.ModuleRevisionId +import core.report.ResolveReport +import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor } +import core.{ IvyPatternHelper, LogOptions } +import org.apache.ivy.util.Message + +private[sbt] object ConsolidatedResolveCache { + def createID(organization: String, name: String, revision: String) = + ModuleRevisionId.newInstance(organization, name, revision) + def sbtOrgTemp = "org.scala-sbt.temp" +} + +private[sbt] class ConsolidatedResolveCache() { + import ConsolidatedResolveCache._ + val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap() + val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap() + val directDependencyCache: concurrent.Map[ModuleDescriptor, Vector[DependencyDescriptor]] = concurrent.TrieMap() + + def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = { + val mrid0 = md0.getModuleRevisionId + val md1 = if (mrid0.getOrganisation == sbtOrgTemp) md0 + else buildConsolidatedModuleDescriptor(md0, prOpt) + val mrid1 = md1.getModuleRevisionId + resolveReportCache.remove(mrid1) + resolvePropertiesCache.remove(mrid1) + } + + def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] = + directDependencyCache.getOrElseUpdate(md0, md0.getDependencies.toVector) + + def buildConsolidatedModuleDescriptor(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): DefaultModuleDescriptor = { + def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] = + prOpt map { + _.getModuleDescriptor(dep.getDependencyRevisionId) match { + case Some(internal) => directDependencies(internal) flatMap expandInternalDeps + case _ => Vector(dep) + } + } getOrElse Vector(dep) + val expanded = directDependencies(md0) flatMap expandInternalDeps + val depStrings = expanded map { dep => + val mrid = dep.getDependencyRevisionId + val confMap = (dep.getModuleConfigurations map { conf => + conf + "->(" + dep.getDependencyConfigurations(conf).mkString(",") + ")" + }) + mrid.toString + ";" + confMap.mkString(";") + } + val depsString = depStrings.distinct.sorted.mkString("\n") + val sha1 = Hash.toHex(Hash(depsString)) + // println("sha1: " + sha1) + val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) + md1 + } +} + +private[sbt] trait ConsolidatedResolveEngine extends ResolveEngine { + import ConsolidatedResolveCache._ + + private[sbt] def consolidatedResolveCache: ConsolidatedResolveCache + private[sbt] def projectResolver: Option[ProjectResolver] + + /** + * Resolve dependencies of a module described by a module descriptor. + */ + override def resolve(md0: ModuleDescriptor, options0: ResolveOptions): ResolveReport = { + val cache = consolidatedResolveCache + val cacheManager = getSettings.getResolutionCacheManager + val md1 = cache.buildConsolidatedModuleDescriptor(md0, projectResolver) + val md1mrid = md1.getModuleRevisionId + + def doWork: (ResolveReport, String) = { + if (options0.getLog != LogOptions.LOG_QUIET) { + Message.info("Consolidating managed dependencies to " + md1mrid.toString + " ...") + } + md1.setLastModified(System.currentTimeMillis) + for { + x <- md0.getConfigurations + } yield md1.addConfiguration(x) + + for { + x <- md0.getDependencies + } yield md1.addDependency(x) + + val options1 = new ResolveOptions(options0) + options1.setOutputReport(false) + val report0 = super.resolve(md1, options1) + val ivyPropertiesInCache1 = cacheManager.getResolvedIvyPropertiesInCache(md1.getResolvedModuleRevisionId) + val prop0 = + if (ivyPropertiesInCache1.exists) IO.read(ivyPropertiesInCache1) + else "" + if (options0.isOutputReport) { + this.outputReport(report0, cacheManager, options0) + } + cache.resolveReportCache(md1mrid) = report0 + cache.resolvePropertiesCache(md1mrid) = prop0 + (report0, prop0) + } + + val (report0, prop0) = (cache.resolveReportCache.get(md1mrid), cache.resolvePropertiesCache.get(md1mrid)) match { + case (Some(report), Some(prop)) => + if (options0.getLog != LogOptions.LOG_QUIET) { + Message.info("Found consolidated dependency " + md1mrid.toString + " ...") + } + (report, prop) + case _ => doWork + } + cacheManager.saveResolvedModuleDescriptor(md0) + if (prop0 != "") { + val ivyPropertiesInCache0 = cacheManager.getResolvedIvyPropertiesInCache(md0.getResolvedModuleRevisionId) + IO.write(ivyPropertiesInCache0, prop0) + } + report0 + } +}