From 8c993d2d531381585a07c3a3e81c84826fa9f893 Mon Sep 17 00:00:00 2001 From: jvican Date: Wed, 26 Apr 2017 23:34:47 +0200 Subject: [PATCH] Fix sbt/sbt#2982: Add a parallel Ivy engine This is a port of https://github.com/sbt/sbt/pull/2992. Original description of the feature: ``` Ivy downloads have traditionally been single-threaded. Parallel downloads are a must for a modern build tool. This commit builds upon the work done by Josh Suereth in the branch sbt/ivy-parallel-download-artifact. To avoid adding external dependencies, it uses the Scala parallel collections. If maintainers consider that is worth it to use a more modern and appropriate approach, like Task, I'm happy to reimplement the features with it. ``` This commit does not preserve Josh's metadata in the commit since the whole design of the repository has changed and I did not know how to import a commit from sbt/sbt. However, it does apply several changes to the original PR. Co-authored-by: Josh Suereth --- .../sbt/internal/librarymanagement/Ivy.scala | 92 ++++++++++------ .../ivyint/ParallelResolveEngine.scala | 102 ++++++++++++++++++ 2 files changed, 160 insertions(+), 34 deletions(-) create mode 100644 librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/ParallelResolveEngine.scala diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala index b5084ffb6..f10561d3a 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala @@ -12,17 +12,17 @@ import org.apache.ivy.core.IvyPatternHelper import org.apache.ivy.core.cache.{ CacheMetadataOptions, DefaultRepositoryCacheManager } import org.apache.ivy.core.event.EventManager import org.apache.ivy.core.module.descriptor.{ - Artifact => IArtifact, DefaultArtifact, DefaultDependencyArtifactDescriptor, - MDArtifact + MDArtifact, + Artifact => IArtifact } import org.apache.ivy.core.module.descriptor.{ DefaultDependencyDescriptor, DefaultModuleDescriptor, DependencyDescriptor, - ModuleDescriptor, - License + License, + ModuleDescriptor } import org.apache.ivy.core.module.descriptor.OverrideDependencyDescriptorMediator import org.apache.ivy.core.module.id.{ ModuleId, ModuleRevisionId } @@ -36,13 +36,13 @@ import org.apache.ivy.util.extendable.ExtendableItem import scala.xml.NodeSeq import scala.collection.mutable - import sbt.util.Logger import sbt.librarymanagement._ import Resolver.PluginPattern import ivyint.{ - CachedResolutionResolveEngine, CachedResolutionResolveCache, + CachedResolutionResolveEngine, + ParallelResolveEngine, SbtDefaultDependencyDescriptor } @@ -101,35 +101,59 @@ final class IvySbt(val configuration: IvyConfiguration) { self => } is } - private[sbt] def mkIvy: Ivy = { - val i = new Ivy() { - private val loggerEngine = new SbtMessageLoggerEngine - override def getLoggerEngine = loggerEngine - override def bind(): Unit = { - val prOpt = Option(getSettings.getResolver(ProjectResolver.InterProject)) map { - case pr: ProjectResolver => pr - } - // We inject the deps we need before we can hook our resolve engine. - setSortEngine(new SortEngine(getSettings)) - setEventManager(new EventManager()) - if (configuration.updateOptions.cachedResolution) { - setResolveEngine( - new ResolveEngine(getSettings, getEventManager, getSortEngine) - with CachedResolutionResolveEngine { - val cachedResolutionResolveCache = IvySbt.cachedResolutionResolveCache - val projectResolver = prOpt - def makeInstance = mkIvy - } - ) - } else setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine)) - super.bind() - } - } - i.setSettings(settings) - i.bind() - i.getLoggerEngine.pushLogger(new IvyLoggerInterface(configuration.log)) - i + /** Defines a parallel [[CachedResolutionResolveEngine]]. + * + * This is defined here because it needs access to [[mkIvy]]. + */ + private class ParallelCachedResolutionResolveEngine( + settings: IvySettings, + eventManager: EventManager, + sortEngine: SortEngine + ) extends ParallelResolveEngine(settings, eventManager, sortEngine) + with CachedResolutionResolveEngine { + def makeInstance: Ivy = mkIvy + val cachedResolutionResolveCache: CachedResolutionResolveCache = + IvySbt.cachedResolutionResolveCache + val projectResolver: Option[ProjectResolver] = { + val res = settings.getResolver(ProjectResolver.InterProject) + Option(res.asInstanceOf[ProjectResolver]) + } + } + + /** Provides a default ivy implementation that decides which resolution + * engine to use depending on the passed ivy configuration options. */ + private class IvyImplementation extends Ivy { + private val loggerEngine = new SbtMessageLoggerEngine + override def getLoggerEngine: SbtMessageLoggerEngine = loggerEngine + override def bind(): Unit = { + val settings = getSettings + val eventManager = new EventManager() + val sortEngine = new SortEngine(settings) + + // We inject the deps we need before we can hook our resolve engine. + setSortEngine(sortEngine) + setEventManager(eventManager) + + val resolveEngine = { + // Decide to use cached resolution if user enabled it + if (configuration.updateOptions.cachedResolution) + new ParallelCachedResolutionResolveEngine(settings, eventManager, sortEngine) + else new ParallelResolveEngine(settings, eventManager, sortEngine) + } + + setResolveEngine(resolveEngine) + super.bind() + } + } + + private[sbt] def mkIvy: Ivy = { + val ivy = new IvyImplementation() + ivy.setSettings(settings) + ivy.bind() + val logger = new IvyLoggerInterface(configuration.log) + ivy.getLoggerEngine.pushLogger(logger) + ivy } private lazy val ivy: Ivy = mkIvy diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/ParallelResolveEngine.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/ParallelResolveEngine.scala new file mode 100644 index 000000000..4e3b4bedd --- /dev/null +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/ParallelResolveEngine.scala @@ -0,0 +1,102 @@ +package sbt.internal.librarymanagement.ivyint + +import org.apache.ivy.core.event.EventManager +import org.apache.ivy.core.event.download.PrepareDownloadEvent +import org.apache.ivy.core.module.descriptor.Artifact +import org.apache.ivy.core.report._ +import org.apache.ivy.core.resolve._ +import org.apache.ivy.core.sort.SortEngine +import org.apache.ivy.util.Message +import org.apache.ivy.util.filter.Filter + +import scala.collection.parallel.mutable.ParArray + +private[ivyint] case class DownloadResult(dep: IvyNode, + report: DownloadReport, + totalSizeDownloaded: Long) + +/** Define an ivy [[ResolveEngine]] that resolves dependencies in parallel. */ +private[sbt] class ParallelResolveEngine(settings: ResolveEngineSettings, + eventManager: EventManager, + sortEngine: SortEngine) + extends ResolveEngine(settings, eventManager, sortEngine) { + + override def downloadArtifacts(report: ResolveReport, + artifactFilter: Filter, + options: DownloadOptions): Unit = { + + val start = System.currentTimeMillis + val dependencies0 = report.getDependencies + val dependencies = dependencies0 + .toArray(new Array[IvyNode](dependencies0.size)) + val artifacts = report.getArtifacts + .toArray(new Array[Artifact](report.getArtifacts.size)) + + getEventManager.fireIvyEvent(new PrepareDownloadEvent(artifacts)) + + // Farm out the dependencies for parallel download + val allDownloads = dependencies.par.flatMap { dep => + if (!(dep.isCompletelyEvicted || dep.hasProblem) && + dep.getModuleRevision != null) { + ParArray(downloadNodeArtifacts(dep, artifactFilter, options)) + } else ParArray.empty[DownloadResult] + } + + // Force parallel downloads and compute total downloaded size + val totalSize = allDownloads.toArray.foldLeft(0L) { + case (size, download) => + val dependency = download.dep + val moduleConfigurations = dependency.getRootModuleConfigurations + moduleConfigurations.foreach { configuration => + val configurationReport = report.getConfigurationReport(configuration) + + // Take into account artifacts required by the given configuration + if (dependency.isEvicted(configuration) || + dependency.isBlacklisted(configuration)) { + configurationReport.addDependency(dependency) + } else configurationReport.addDependency(dependency, download.report) + } + + size + download.totalSizeDownloaded + } + + report.setDownloadTime(System.currentTimeMillis() - start) + report.setDownloadSize(totalSize) + } + + /** + * Download all the artifacts associated with an ivy node. + * + * Return the report and the total downloaded size. + */ + private def downloadNodeArtifacts(dependency: IvyNode, + artifactFilter: Filter, + options: DownloadOptions): DownloadResult = { + + val resolver = dependency.getModuleRevision.getArtifactResolver + val selectedArtifacts = dependency.getSelectedArtifacts(artifactFilter) + val downloadReport = resolver.download(selectedArtifacts, options) + val artifactReports = downloadReport.getArtifactsReports + + val totalSize = artifactReports.foldLeft(0L) { (size, artifactReport) => + // Check download status and report resolution failures + artifactReport.getDownloadStatus match { + case DownloadStatus.SUCCESSFUL => + size + artifactReport.getSize + case DownloadStatus.FAILED => + val artifact = artifactReport.getArtifact + val mergedAttribute = artifact.getExtraAttribute("ivy:merged") + if (mergedAttribute != null) { + Message.warn(s"\tMissing merged artifact: $artifact, required by $mergedAttribute.") + } else { + Message.warn(s"\tDetected merged artifact: $artifactReport.") + resolver.reportFailure(artifactReport.getArtifact) + } + size + case _ => size + } + } + + DownloadResult(dependency, downloadReport, totalSize) + } +}