Merge pull request #90 from scalacenter/add-parallel-ivy

Fix sbt/sbt#2982: Add a parallel Ivy engine

librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala

@@ -12,17 +12,17 @@ import org.apache.ivy.core.IvyPatternHelper
 import org.apache.ivy.core.cache.{ CacheMetadataOptions, DefaultRepositoryCacheManager }
 import org.apache.ivy.core.event.EventManager
 import org.apache.ivy.core.module.descriptor.{
-  Artifact => IArtifact,
   DefaultArtifact,
   DefaultDependencyArtifactDescriptor,
-  MDArtifact
+  MDArtifact,
+  Artifact => IArtifact
 }
 import org.apache.ivy.core.module.descriptor.{
   DefaultDependencyDescriptor,
   DefaultModuleDescriptor,
   DependencyDescriptor,
-  ModuleDescriptor,
-  License
+  License,
+  ModuleDescriptor
 }
 import org.apache.ivy.core.module.descriptor.OverrideDependencyDescriptorMediator
 import org.apache.ivy.core.module.id.{ ModuleId, ModuleRevisionId }
@@ -36,13 +36,13 @@ import org.apache.ivy.util.extendable.ExtendableItem
 
 import scala.xml.NodeSeq
 import scala.collection.mutable
-
 import sbt.util.Logger
 import sbt.librarymanagement._
 import Resolver.PluginPattern
 import ivyint.{
-  CachedResolutionResolveEngine,
   CachedResolutionResolveCache,
+  CachedResolutionResolveEngine,
+  ParallelResolveEngine,
   SbtDefaultDependencyDescriptor
 }
 
@@ -101,35 +101,59 @@ final class IvySbt(val configuration: IvyConfiguration) { self =>
     }
     is
   }
-  private[sbt] def mkIvy: Ivy = {
-    val i = new Ivy() {
-      private val loggerEngine = new SbtMessageLoggerEngine
-      override def getLoggerEngine = loggerEngine
-      override def bind(): Unit = {
-        val prOpt = Option(getSettings.getResolver(ProjectResolver.InterProject)) map {
-          case pr: ProjectResolver => pr
-        }
-        // We inject the deps we need before we can hook our resolve engine.
-        setSortEngine(new SortEngine(getSettings))
-        setEventManager(new EventManager())
-        if (configuration.updateOptions.cachedResolution) {
-          setResolveEngine(
-            new ResolveEngine(getSettings, getEventManager, getSortEngine)
-            with CachedResolutionResolveEngine {
-              val cachedResolutionResolveCache = IvySbt.cachedResolutionResolveCache
-              val projectResolver = prOpt
-              def makeInstance = mkIvy
-            }
-          )
-        } else setResolveEngine(new ResolveEngine(getSettings, getEventManager, getSortEngine))
-        super.bind()
-      }
-    }
 
-    i.setSettings(settings)
-    i.bind()
-    i.getLoggerEngine.pushLogger(new IvyLoggerInterface(configuration.log))
-    i
+  /** Defines a parallel [[CachedResolutionResolveEngine]].
+   *
+   * This is defined here because it needs access to [[mkIvy]].
+   */
+  private class ParallelCachedResolutionResolveEngine(
+      settings: IvySettings,
+      eventManager: EventManager,
+      sortEngine: SortEngine
+  ) extends ParallelResolveEngine(settings, eventManager, sortEngine)
+      with CachedResolutionResolveEngine {
+    def makeInstance: Ivy = mkIvy
+    val cachedResolutionResolveCache: CachedResolutionResolveCache =
+      IvySbt.cachedResolutionResolveCache
+    val projectResolver: Option[ProjectResolver] = {
+      val res = settings.getResolver(ProjectResolver.InterProject)
+      Option(res.asInstanceOf[ProjectResolver])
+    }
+  }
+
+  /** Provides a default ivy implementation that decides which resolution
+   * engine to use depending on the passed ivy configuration options. */
+  private class IvyImplementation extends Ivy {
+    private val loggerEngine = new SbtMessageLoggerEngine
+    override def getLoggerEngine: SbtMessageLoggerEngine = loggerEngine
+    override def bind(): Unit = {
+      val settings = getSettings
+      val eventManager = new EventManager()
+      val sortEngine = new SortEngine(settings)
+
+      // We inject the deps we need before we can hook our resolve engine.
+      setSortEngine(sortEngine)
+      setEventManager(eventManager)
+
+      val resolveEngine = {
+        // Decide to use cached resolution if user enabled it
+        if (configuration.updateOptions.cachedResolution)
+          new ParallelCachedResolutionResolveEngine(settings, eventManager, sortEngine)
+        else new ParallelResolveEngine(settings, eventManager, sortEngine)
+      }
+
+      setResolveEngine(resolveEngine)
+      super.bind()
+    }
+  }
+
+  private[sbt] def mkIvy: Ivy = {
+    val ivy = new IvyImplementation()
+    ivy.setSettings(settings)
+    ivy.bind()
+    val logger = new IvyLoggerInterface(configuration.log)
+    ivy.getLoggerEngine.pushLogger(logger)
+    ivy
   }
 
   private lazy val ivy: Ivy = mkIvy

librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/ParallelResolveEngine.scala

@@ -0,0 +1,102 @@
+package sbt.internal.librarymanagement.ivyint
+
+import org.apache.ivy.core.event.EventManager
+import org.apache.ivy.core.event.download.PrepareDownloadEvent
+import org.apache.ivy.core.module.descriptor.Artifact
+import org.apache.ivy.core.report._
+import org.apache.ivy.core.resolve._
+import org.apache.ivy.core.sort.SortEngine
+import org.apache.ivy.util.Message
+import org.apache.ivy.util.filter.Filter
+
+import scala.collection.parallel.mutable.ParArray
+
+private[ivyint] case class DownloadResult(dep: IvyNode,
+                                          report: DownloadReport,
+                                          totalSizeDownloaded: Long)
+
+/** Define an ivy [[ResolveEngine]] that resolves dependencies in parallel. */
+private[sbt] class ParallelResolveEngine(settings: ResolveEngineSettings,
+                                         eventManager: EventManager,
+                                         sortEngine: SortEngine)
+    extends ResolveEngine(settings, eventManager, sortEngine) {
+
+  override def downloadArtifacts(report: ResolveReport,
+                                 artifactFilter: Filter,
+                                 options: DownloadOptions): Unit = {
+
+    val start = System.currentTimeMillis
+    val dependencies0 = report.getDependencies
+    val dependencies = dependencies0
+      .toArray(new Array[IvyNode](dependencies0.size))
+    val artifacts = report.getArtifacts
+      .toArray(new Array[Artifact](report.getArtifacts.size))
+
+    getEventManager.fireIvyEvent(new PrepareDownloadEvent(artifacts))
+
+    // Farm out the dependencies for parallel download
+    val allDownloads = dependencies.par.flatMap { dep =>
+      if (!(dep.isCompletelyEvicted || dep.hasProblem) &&
+          dep.getModuleRevision != null) {
+        ParArray(downloadNodeArtifacts(dep, artifactFilter, options))
+      } else ParArray.empty[DownloadResult]
+    }
+
+    // Force parallel downloads and compute total downloaded size
+    val totalSize = allDownloads.toArray.foldLeft(0L) {
+      case (size, download) =>
+        val dependency = download.dep
+        val moduleConfigurations = dependency.getRootModuleConfigurations
+        moduleConfigurations.foreach { configuration =>
+          val configurationReport = report.getConfigurationReport(configuration)
+
+          // Take into account artifacts required by the given configuration
+          if (dependency.isEvicted(configuration) ||
+              dependency.isBlacklisted(configuration)) {
+            configurationReport.addDependency(dependency)
+          } else configurationReport.addDependency(dependency, download.report)
+        }
+
+        size + download.totalSizeDownloaded
+    }
+
+    report.setDownloadTime(System.currentTimeMillis() - start)
+    report.setDownloadSize(totalSize)
+  }
+
+  /**
+   * Download all the artifacts associated with an ivy node.
+   *
+   * Return the report and the total downloaded size.
+   */
+  private def downloadNodeArtifacts(dependency: IvyNode,
+                                    artifactFilter: Filter,
+                                    options: DownloadOptions): DownloadResult = {
+
+    val resolver = dependency.getModuleRevision.getArtifactResolver
+    val selectedArtifacts = dependency.getSelectedArtifacts(artifactFilter)
+    val downloadReport = resolver.download(selectedArtifacts, options)
+    val artifactReports = downloadReport.getArtifactsReports
+
+    val totalSize = artifactReports.foldLeft(0L) { (size, artifactReport) =>
+      // Check download status and report resolution failures
+      artifactReport.getDownloadStatus match {
+        case DownloadStatus.SUCCESSFUL =>
+          size + artifactReport.getSize
+        case DownloadStatus.FAILED =>
+          val artifact = artifactReport.getArtifact
+          val mergedAttribute = artifact.getExtraAttribute("ivy:merged")
+          if (mergedAttribute != null) {
+            Message.warn(s"\tMissing merged artifact: $artifact, required by $mergedAttribute.")
+          } else {
+            Message.warn(s"\tDetected merged artifact: $artifactReport.")
+            resolver.reportFailure(artifactReport.getArtifact)
+          }
+          size
+        case _ => size
+      }
+    }
+
+    DownloadResult(dependency, downloadReport, totalSize)
+  }
+}