Merge pull request #462 from coursier/topic/profiling

Profiling...
2017-04-24 23:10:00 +02:00 · 2017-04-24 23:10:00 +02:00 · e6c28f02d6
parent 916488393a 34e591792a
commit e6c28f02d6
7 changed files with 105 additions and 43 deletions
--- a/README.md
+++ b/README.md
@ -781,5 +781,13 @@ add dependencies on-the-fly ([#4](https://github.com/apache/incubator-toree/pull
 - [Quill](https://github.com/getquill/quill) is using coursier for faster dependency resolution ([#591](https://github.com/getquill/quill/pull/591)),
 - Your project here :-)

+## Acknowledgements
+
+<a href="https://www.ej-technologies.com/products/jprofiler/overview.html">
+  <img src="https://www.ej-technologies.com/images/product_banners/jprofiler_medium.png">
+</a>
+
+Thanks to [JProfiler](https://www.ej-technologies.com/products/jprofiler/overview.html) for having kindly granted an [Open Source license](https://www.ej-technologies.com/buy/jprofiler/openSource)  to help for the development of coursier.
+

 Released under the Apache license, v2.
--- a/core/shared/src/main/scala/coursier/package.scala
+++ b/core/shared/src/main/scala/coursier/package.scala
@ -54,6 +54,7 @@ package object coursier {

  type ModuleVersion = (core.Module, String)

+  type ProjectCache = Map[ModuleVersion, (Artifact.Source, Project)]

  type Repository = core.Repository
  val Repository = core.Repository
@ -69,7 +70,7 @@ package object coursier {
      dependencies: Set[Dependency] = Set.empty,
      forceVersions: Map[Module, String] = Map.empty,
      conflicts: Set[Dependency] = Set.empty,
-      projectCache: Map[ModuleVersion, (Artifact.Source, Project)] = Map.empty,
+      projectCache: ProjectCache = Map.empty,
      errorCache: Map[ModuleVersion, Seq[String]] = Map.empty,
      finalDependencies: Map[Dependency, Seq[Dependency]] = Map.empty,
      filter: Option[Dependency => Boolean] = None,
--- a/doc/README.md
+++ b/doc/README.md
@ -815,5 +815,13 @@ add dependencies on-the-fly ([#4](https://github.com/apache/incubator-toree/pull
 - [Quill](https://github.com/getquill/quill) is using coursier for faster dependency resolution ([#591](https://github.com/getquill/quill/pull/591)),
 - Your project here :-)

+## Acknowledgements
+
+<a href="https://www.ej-technologies.com/products/jprofiler/overview.html">
+  <img src="https://www.ej-technologies.com/images/product_banners/jprofiler_medium.png">
+</a>
+
+Thanks to [JProfiler](https://www.ej-technologies.com/products/jprofiler/overview.html) for having kindly granted an [Open Source license](https://www.ej-technologies.com/buy/jprofiler/openSource)  to help for the development of coursier.
+

 Released under the Apache license, v2.
--- a/sbt-coursier/src/main/scala-2.10/coursier/CoursierPlugin.scala
+++ b/sbt-coursier/src/main/scala-2.10/coursier/CoursierPlugin.scala
@ -32,6 +32,7 @@ object CoursierPlugin extends AutoPlugin {

    val coursierConfigurations = Keys.coursierConfigurations

+    val coursierParentProjectCache = Keys.coursierParentProjectCache
    val coursierResolution = Keys.coursierResolution
    val coursierSbtClassifiersResolution = Keys.coursierSbtClassifiersResolution

@ -118,6 +119,7 @@ object CoursierPlugin extends AutoPlugin {
    coursierPublications <<= Tasks.coursierPublicationsTask(packageConfigs: _*),
    coursierSbtClassifiersModule <<= classifiersModule in updateSbtClassifiers,
    coursierConfigurations <<= Tasks.coursierConfigurationsTask(None),
+    coursierParentProjectCache <<= Tasks.parentProjectCacheTask,
    coursierResolution <<= Tasks.resolutionTask(),
    coursierSbtClassifiersResolution <<= Tasks.resolutionTask(
      sbtClassifiers = true
--- a/sbt-coursier/src/main/scala-2.10/coursier/Keys.scala
+++ b/sbt-coursier/src/main/scala-2.10/coursier/Keys.scala
@ -40,6 +40,8 @@ object Keys {

  val coursierConfigurations = TaskKey[Map[String, Set[String]]]("coursier-configurations")

+
+  val coursierParentProjectCache = TaskKey[Map[Seq[Resolver], Seq[ProjectCache]]]("coursier-parent-project-cache")
  val coursierResolution = TaskKey[Resolution]("coursier-resolution")
  val coursierSbtClassifiersResolution = TaskKey[Resolution]("coursier-sbt-classifiers-resolution")

--- a/sbt-coursier/src/main/scala-2.10/coursier/Tasks.scala
+++ b/sbt-coursier/src/main/scala-2.10/coursier/Tasks.scala
@ -1,27 +1,25 @@
 package coursier

-import java.io.{ File, InputStream, OutputStreamWriter }
+import java.io.{File, InputStream, OutputStreamWriter}
 import java.net.URL
-import java.util.concurrent.{ ExecutorService, Executors }
+import java.util.concurrent.{ExecutorService, Executors}

-import coursier.core.{ Authentication, Publication }
-import coursier.ivy.{ IvyRepository, PropertiesPattern }
+import coursier.core.{Authentication, Publication}
+import coursier.ivy.{IvyRepository, PropertiesPattern}
 import coursier.Keys._
 import coursier.Structure._
 import coursier.internal.FileUtil
-import coursier.util.{ Config, Print }
+import coursier.util.{Config, Print}
 import org.apache.ivy.core.module.id.ModuleRevisionId
-
-import sbt.{ UpdateReport, Classpaths, Resolver, Def }
+import sbt.{ClasspathDep, Classpaths, Def, ProjectRef, Resolver, UpdateReport}
 import sbt.Keys._

 import scala.collection.mutable
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.util.Try
-
-import scalaz.{ \/-, -\/ }
-import scalaz.concurrent.{ Task, Strategy }
+import scalaz.{-\/, \/-}
+import scalaz.concurrent.{Strategy, Task}

 object Tasks {

@ -364,6 +362,28 @@ object Tasks {
    }
  }

+  def parentProjectCacheTask: Def.Initialize[sbt.Task[Map[Seq[sbt.Resolver],Seq[coursier.ProjectCache]]]] =
+    (sbt.Keys.state,
+      sbt.Keys.thisProjectRef).flatMap{ (state, projectRef) =>
+
+      val projectDeps = structure(state).allProjects
+        .find(_.id == projectRef.project)
+        .map(_.dependencies.map(_.project.project).toSet)
+        .getOrElse(Set.empty)
+
+      val projects = structure(state).allProjectRefs.filter(p => projectDeps(p.project))
+
+      coursierRecursiveResolvers.forAllProjects(state, projects).flatMap{ m =>
+        coursierResolution.forAllProjects(state, m.keys.toSeq).map{ n =>
+          n.foldLeft(Map.empty[Seq[Resolver], Seq[ProjectCache]]){ case (caches, (ref, resolution)) =>
+            m.get(ref).fold(caches)(resolvers =>
+              caches.updated(resolvers, resolution.projectCache +: caches.getOrElse(resolvers, Seq.empty)))
+          }
+        }
+      }
+    }
+
+
  def resolutionTask(
    sbtClassifiers: Boolean = false
  ) = Def.task {
@ -447,6 +467,11 @@ object Tasks {
        else
          coursierRecursiveResolvers.value.distinct

+      val parentProjectCache: ProjectCache = coursierParentProjectCache.value
+          .get(resolvers)
+          .map(_.foldLeft[ProjectCache](Map.empty)(_ ++ _))
+          .getOrElse(Map.empty)
+
      // TODO Warn about possible duplicated modules from source repositories?

      val verbosityLevel = coursierVerbosity.value
@ -468,7 +493,8 @@ object Tasks {
          // order matters here
          userForceVersions ++
          forcedScalaModules(so, sv) ++
-          interProjectDependencies.map(_.moduleVersion)
+          interProjectDependencies.map(_.moduleVersion),
+        projectCache = parentProjectCache
      )

      if (verbosityLevel >= 2) {
--- a/sbt-coursier/src/main/scala-2.10/coursier/ToSbt.scala
+++ b/sbt-coursier/src/main/scala-2.10/coursier/ToSbt.scala
@ -1,6 +1,7 @@
 package coursier

 import java.util.GregorianCalendar
+import java.util.concurrent.ConcurrentHashMap

 import coursier.maven.MavenSource

@ -8,43 +9,57 @@ import sbt._

 object ToSbt {

-  def moduleId(dependency: Dependency, extraProperties: Seq[(String, String)]): sbt.ModuleID =
-    sbt.ModuleID(
-      dependency.module.organization,
-      dependency.module.name,
-      dependency.version,
-      configurations = Some(dependency.configuration),
-      extraAttributes = dependency.module.attributes ++ extraProperties
-    )
+  private def caching[K, V](f: K => V): K => V = {

-  def artifact(module: Module, artifact: Artifact, extraProperties: Seq[(String, String)]): sbt.Artifact =
-    sbt.Artifact(
-      module.name,
-      // FIXME Get these two from publications
-      artifact.attributes.`type`,
-      MavenSource.typeExtension(artifact.attributes.`type`),
-      Some(artifact.attributes.classifier)
-        .filter(_.nonEmpty)
-        .orElse(MavenSource.typeDefaultClassifierOpt(artifact.attributes.`type`)),
-      Nil,
-      Some(url(artifact.url)),
-      module.attributes ++ extraProperties
-    )
+    val cache = new ConcurrentHashMap[K, V]

-  def moduleReport(
-    dependency: Dependency,
-    dependees: Seq[(Dependency, Project)],
-    project: Project,
-    artifacts: Seq[(Artifact, Option[File])]
-  ): sbt.ModuleReport = {
+    key =>
+      val previousValueOpt = Option(cache.get(key))
+
+      previousValueOpt.getOrElse {
+        val value = f(key)
+        val concurrentValueOpt = Option(cache.putIfAbsent(key, value))
+        concurrentValueOpt.getOrElse(value)
+      }
+  }
+
+  val moduleId = caching[(Dependency, Map[String, String]), sbt.ModuleID] {
+    case (dependency, extraProperties) =>
+      sbt.ModuleID(
+        dependency.module.organization,
+        dependency.module.name,
+        dependency.version,
+        configurations = Some(dependency.configuration),
+        extraAttributes = dependency.module.attributes ++ extraProperties
+      )
+  }
+
+  val artifact = caching[(Module, Map[String, String], Artifact), sbt.Artifact] {
+    case (module, extraProperties, artifact) =>
+      sbt.Artifact(
+        module.name,
+        // FIXME Get these two from publications
+        artifact.attributes.`type`,
+        MavenSource.typeExtension(artifact.attributes.`type`),
+        Some(artifact.attributes.classifier)
+          .filter(_.nonEmpty)
+          .orElse(MavenSource.typeDefaultClassifierOpt(artifact.attributes.`type`)),
+        Nil,
+        Some(url(artifact.url)),
+        module.attributes ++ extraProperties
+      )
+  }
+
+  val moduleReport = caching[(Dependency, Seq[(Dependency, Project)], Project, Seq[(Artifact, Option[File])]), sbt.ModuleReport] {
+    case (dependency, dependees, project, artifacts) =>

    val sbtArtifacts = artifacts.collect {
      case (artifact, Some(file)) =>
-        (ToSbt.artifact(dependency.module, artifact, project.properties), file)
+        (ToSbt.artifact(dependency.module, project.properties.toMap, artifact), file)
    }
    val sbtMissingArtifacts = artifacts.collect {
      case (artifact, None) =>
-        ToSbt.artifact(dependency.module, artifact, project.properties)
+        ToSbt.artifact(dependency.module, project.properties.toMap, artifact)
    }

    val publicationDate = project.info.publication.map { dt =>
@ -54,7 +69,7 @@ object ToSbt {
    val callers = dependees.map {
      case (dependee, dependeeProj) =>
        new Caller(
-          ToSbt.moduleId(dependee, dependeeProj.properties),
+          ToSbt.moduleId(dependee, dependeeProj.properties.toMap),
          dependeeProj.configurations.keys.toVector,
          dependee.module.attributes ++ dependeeProj.properties,
          // FIXME Set better values here
@ -66,7 +81,7 @@ object ToSbt {
    }

    new sbt.ModuleReport(
-      module = ToSbt.moduleId(dependency, project.properties),
+      module = ToSbt.moduleId(dependency, project.properties.toMap),
      artifacts = sbtArtifacts,
      missingArtifacts = sbtMissingArtifacts,
      status = None,