Run separate resolutions for distinct Ivy configurations subgraphs

sbt-coursier/src/main/scala/coursier/CoursierPlugin.scala

@@ -28,6 +28,7 @@ object CoursierPlugin extends AutoPlugin {
     val coursierFallbackDependencies = Keys.coursierFallbackDependencies
     val coursierCache = Keys.coursierCache
     val coursierProject = Keys.coursierProject
+    val coursierConfigGraphs = Keys.coursierConfigGraphs
     val coursierInterProjectDependencies = Keys.coursierInterProjectDependencies
     val coursierPublications = Keys.coursierPublications
     val coursierSbtClassifiersModule = Keys.coursierSbtClassifiersModule
@@ -35,7 +36,11 @@ object CoursierPlugin extends AutoPlugin {
     val coursierConfigurations = Keys.coursierConfigurations
 
     val coursierParentProjectCache = Keys.coursierParentProjectCache
-    val coursierResolution = Keys.coursierResolution
+    val coursierResolutions = Keys.coursierResolutions
+
+    @deprecated("Use coursierResolutions instead", "1.0.0-RC4")
+    val coursierResolution = Keys.actualCoursierResolution
+
     val coursierSbtClassifiersResolution = Keys.coursierSbtClassifiersResolution
 
     val coursierDependencyTree = Keys.coursierDependencyTree
@@ -117,15 +122,30 @@ object CoursierPlugin extends AutoPlugin {
       ignoreArtifactErrors = true
     ).value,
     coursierProject := Tasks.coursierProjectTask.value,
+    coursierConfigGraphs := Tasks.ivyGraphsTask.value,
     coursierInterProjectDependencies := Tasks.coursierInterProjectDependenciesTask.value,
     coursierPublications := Tasks.coursierPublicationsTask(packageConfigs: _*).value,
     coursierSbtClassifiersModule := classifiersModule.in(updateSbtClassifiers).value,
     coursierConfigurations := Tasks.coursierConfigurationsTask(None).value,
     coursierParentProjectCache := Tasks.parentProjectCacheTask.value,
-    coursierResolution := Tasks.resolutionTask().value,
+    coursierResolutions := Tasks.resolutionsTask().value,
-    coursierSbtClassifiersResolution := Tasks.resolutionTask(
+    Keys.actualCoursierResolution := {
+
+      val config = Compile.name
+
+      coursierResolutions
+        .value
+        .collectFirst {
+          case (configs, res) if (configs(config)) =>
+            res
+        }
+        .getOrElse {
+          sys.error(s"Resolution for configuration $config not found")
+        }
+    },
+    coursierSbtClassifiersResolution := Tasks.resolutionsTask(
       sbtClassifiers = true
-    ).value,
+    ).value.head._2,
     ivyConfigurations := {
       val confs = ivyConfigurations.value
       val names = confs.map(_.name).toSet

sbt-coursier/src/main/scala/coursier/Keys.scala

@@ -34,6 +34,7 @@ object Keys {
   val coursierFallbackDependencies = TaskKey[Seq[(Module, String, URL, Boolean)]]("coursier-fallback-dependencies")
 
   val coursierProject = TaskKey[Project]("coursier-project")
+  val coursierConfigGraphs = TaskKey[Seq[Set[String]]]("coursier-config-graphs")
   val coursierInterProjectDependencies = TaskKey[Seq[Project]]("coursier-inter-project-dependencies", "Projects the current project depends on, possibly transitively")
   val coursierPublications = TaskKey[Seq[(String, Publication)]]("coursier-publications")
 
@@ -43,7 +44,12 @@ object Keys {
 
 
   val coursierParentProjectCache = TaskKey[Map[Seq[Resolver], Seq[ProjectCache]]]("coursier-parent-project-cache")
-  val coursierResolution = TaskKey[Resolution]("coursier-resolution")
+  val coursierResolutions = TaskKey[Map[Set[String], Resolution]]("coursier-resolutions")
+
+  private[coursier] val actualCoursierResolution = TaskKey[Resolution]("coursier-resolution")
+
+  @deprecated("Use coursierResolutions instead", "1.0.0-RC4")
+  val coursierResolution = actualCoursierResolution
   val coursierSbtClassifiersResolution = TaskKey[Resolution]("coursier-sbt-classifiers-resolution")
 
   val coursierDependencyTree = TaskKey[Unit](

sbt-coursier/src/main/scala/coursier/Tasks.scala

@@ -9,7 +9,7 @@ import coursier.extra.Typelevel
 import coursier.ivy.{IvyRepository, PropertiesPattern}
 import coursier.Keys._
 import coursier.Structure._
-import coursier.util.{Config, Print}
+import coursier.util.Print
 import sbt.{Classpaths, Def, Resolver, UpdateReport}
 import sbt.Keys._
 
@@ -344,18 +344,18 @@ object Tasks {
     project: Project,
     repositories: Seq[Repository],
     userEnabledProfiles: Set[String],
-    resolution: Resolution,
+    resolution: Map[Set[String], Resolution],
     sbtClassifiers: Boolean
   )
 
   private final case class ReportCacheKey(
     project: Project,
-    resolution: Resolution,
+    resolution: Map[Set[String], Resolution],
     withClassifiers: Boolean,
     sbtClassifiers: Boolean
   )
 
-  private val resolutionsCache = new mutable.HashMap[ResolutionCacheKey, Resolution]
+  private val resolutionsCache = new mutable.HashMap[ResolutionCacheKey, Map[Set[String], Resolution]]
   // these may actually not need to be cached any more, now that the resolutions
   // are cached
   private val reportsCache = new mutable.HashMap[ReportCacheKey, UpdateReport]
@@ -423,28 +423,76 @@ object Tasks {
       val t =
         for {
           m <- coursierRecursiveResolvers.forAllProjects(state, projects)
-          n <- coursierResolution.forAllProjects(state, m.keys.toSeq)
+          n <- coursierResolutions.forAllProjects(state, m.keys.toSeq)
         } yield
           n.foldLeft(Map.empty[Seq[Resolver], Seq[ProjectCache]]) {
-            case (caches, (ref, resolution)) =>
+            case (caches, (ref, resolutions)) =>
-              m.get(ref).fold(caches) { resolvers =>
+              val mainResOpt = resolutions.collectFirst {
-                caches.updated(
+                case (k, v) if k("compile") => v
-                  resolvers,
-                  resolution.projectCache +: caches.getOrElse(resolvers, Nil)
-                )
               }
+
+              val r = for {
+                resolvers <- m.get(ref)
+                resolution <- mainResOpt
+              } yield
+                caches.updated(resolvers, resolution.projectCache +: caches.getOrElse(resolvers, Seq.empty))
+
+              r.getOrElse(caches)
           }
 
       Def.task(t.value)
     }
 
+
+  def ivyGraphsTask = Def.task {
+
+    // probably bad complexity, but that shouldn't matter given the size of the graphs involved...
+
+    val p = coursierProject.value
+
+    final class Wrapper(val set: mutable.HashSet[String]) {
+      def ++=(other: Wrapper): this.type = {
+        set ++= other.set
+        this
+      }
+    }
+
+    val sets =
+      new mutable.HashMap[String, Wrapper] ++= p.configurations.map {
+        case (k, l) =>
+          val s = new mutable.HashSet[String]()
+          s ++= l
+          s += k
+          k -> new Wrapper(s)
+      }
+
+    for (k <- p.configurations.keys) {
+      val s = sets(k)
+
+      var foundNew = true
+      while (foundNew) {
+        foundNew = false
+        for (other <- s.set.toVector) {
+          val otherS = sets(other)
+          if (!otherS.eq(s)) {
+            s ++= otherS
+            sets += other -> s
+            foundNew = true
+          }
+        }
+      }
+    }
+
+    sets.values.toVector.distinct.map(_.set.toSet)
+  }
+
   private val noOptionalFilter: Option[Dependency => Boolean] = Some(dep => !dep.optional)
   private val typelevelOrgSwap: Option[Dependency => Dependency] = Some(Typelevel.swap(_))
 
 
-  def resolutionTask(
+  def resolutionsTask(
     sbtClassifiers: Boolean = false
-  ): Def.Initialize[sbt.Task[coursier.Resolution]] = Def.task {
+  ): Def.Initialize[sbt.Task[Map[Set[String], coursier.Resolution]]] = Def.task {
 
     // let's update only one module at once, for a better output
     // Downloads are already parallel, no need to parallelize further anyway
@@ -454,7 +502,7 @@ object Tasks {
 
       lazy val projectName = thisProjectRef.value.project
 
-      val (currentProject, fallbackDependencies) =
+      val (currentProject, fallbackDependencies, configGraphs) =
         if (sbtClassifiers) {
           val sv = scalaVersion.value
           val sbv = scalaBinaryVersion.value
@@ -473,12 +521,12 @@ object Tasks {
             sbv
           )
 
-          (proj, fallbackDeps)
+          (proj, fallbackDeps, Vector(cm.configurations.map(_.name).toSet))
         } else {
           val proj = coursierProject.value
           val publications = coursierPublications.value
           val fallbackDeps = coursierFallbackDependencies.value
-          (proj.copy(publications = publications), fallbackDeps)
+          (proj.copy(publications = publications), fallbackDeps, coursierConfigGraphs.value)
         }
 
       val interProjectDependencies = coursierInterProjectDependencies.value
@@ -520,23 +568,6 @@ object Tasks {
 
       val typelevel = scalaOrganization.value == Typelevel.typelevelOrg
 
-      val startRes = Resolution(
-        currentProject.dependencies.map(_._2).toSet,
-        filter = noOptionalFilter,
-        userActivations =
-          if (userEnabledProfiles.isEmpty)
-            None
-          else
-            Some(userEnabledProfiles.iterator.map(_ -> true).toMap),
-        forceVersions =
-          // order matters here
-          userForceVersions ++
-          forcedScalaModules(so, sv) ++
-          interProjectDependencies.map(_.moduleVersion),
-        projectCache = parentProjectCache,
-        mapDependencies = if (typelevel) typelevelOrgSwap else None
-      )
-
       if (verbosityLevel >= 2) {
         log.info("InterProjectRepository")
         for (p <- interProjectDependencies)
@@ -653,7 +684,31 @@ object Tasks {
         }.map(withAuthenticationByHost(_, authenticationByHost)) ++
         fallbackDependenciesRepositories
 
-      def resolution = {
+      def startRes(configs: Set[String]) = Resolution(
+        currentProject
+          .dependencies
+          .collect {
+            case (config, dep) if configs(config) =>
+              dep
+          }
+          .toSet,
+        filter = noOptionalFilter,
+        userActivations =
+          if (userEnabledProfiles.isEmpty)
+            None
+          else
+            Some(userEnabledProfiles.iterator.map(_ -> true).toMap),
+        forceVersions =
+          // order matters here
+          userForceVersions ++
+            (if (configs("compile") || configs("scala-tool")) forcedScalaModules(so, sv) else Map()) ++
+            interProjectDependencies.map(_.moduleVersion),
+        projectCache = parentProjectCache,
+        mapDependencies = if (typelevel && (configs("compile") || configs("scala-tool"))) typelevelOrgSwap else None
+      )
+
+      def resolution(startRes: Resolution) = {
+
         var pool: ExecutorService = null
         var resLogger: TermDisplay = null
 
@@ -763,15 +818,20 @@ object Tasks {
         res
       }
 
+      val allStartRes = configGraphs.map(configs => configs -> startRes(configs)).toMap
+
       resolutionsCache.getOrElseUpdate(
         ResolutionCacheKey(
           currentProject,
           repositories,
           userEnabledProfiles,
-          startRes.copy(filter = None),
+          allStartRes,
           sbtClassifiers
         ),
-        resolution
+        allStartRes.map {
+          case (config, startRes) =>
+            config -> resolution(startRes)
+        }
       )
     }
   }
@@ -800,12 +860,11 @@ object Tasks {
 
       val verbosityLevel = coursierVerbosity.value
 
-      val res = {
+      val res =
         if (withClassifiers && sbtClassifiers)
-          coursierSbtClassifiersResolution
+          Seq(coursierSbtClassifiersResolution.value)
         else
-          coursierResolution
+          coursierResolutions.value.values.toVector
-      }.value
 
       val classifiers =
         if (withClassifiers)
@@ -820,8 +879,8 @@ object Tasks {
 
       val allArtifacts =
         classifiers match {
-          case None => res.artifacts
+          case None => res.flatMap(_.artifacts)
-          case Some(cl) => res.classifiersArtifacts(cl)
+          case Some(cl) => res.flatMap(_.classifiersArtifacts(cl))
         }
 
       var pool: ExecutorService = null
@@ -833,7 +892,7 @@ object Tasks {
         pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory)
         artifactsLogger = createLogger()
 
-        val artifactFileOrErrorTasks = allArtifacts.toVector.map { a =>
+        val artifactFileOrErrorTasks = allArtifacts.toVector.distinct.map { a =>
           def f(p: CachePolicy) =
             Cache.file(
               a,
@@ -916,6 +975,48 @@ object Tasks {
     res
   }
 
+  // Move back to coursier.util (in core module) after 1.0?
+  private def allDependenciesByConfig(
+    res: Map[String, Resolution],
+    depsByConfig: Map[String, Set[Dependency]],
+    configs: Map[String, Set[String]]
+  ): Map[String, Set[Dependency]] = {
+
+    val allDepsByConfig = depsByConfig.map {
+      case (config, deps) =>
+        config -> res(config).subset(deps).minDependencies
+    }
+
+    val filteredAllDepsByConfig = allDepsByConfig.map {
+      case (config, allDeps) =>
+        val allExtendedConfigs = configs.getOrElse(config, Set.empty) - config
+        val inherited = allExtendedConfigs
+          .flatMap(allDepsByConfig.getOrElse(_, Set.empty))
+
+        config -> (allDeps -- inherited)
+    }
+
+    filteredAllDepsByConfig
+  }
+
+  // Move back to coursier.util (in core module) after 1.0?
+  private def dependenciesWithConfig(
+    res: Map[String, Resolution],
+    depsByConfig: Map[String, Set[Dependency]],
+    configs: Map[String, Set[String]]
+  ): Set[Dependency] =
+    allDependenciesByConfig(res, depsByConfig, configs)
+      .flatMap {
+        case (config, deps) =>
+          deps.map(dep => dep.copy(configuration = s"$config->${dep.configuration}"))
+      }
+      .groupBy(_.copy(configuration = ""))
+      .map {
+        case (dep, l) =>
+          dep.copy(configuration = l.map(_.configuration).mkString(";"))
+      }
+      .toSet
+
   def updateTask(
     shadedConfigOpt: Option[(String, String)],
     withClassifiers: Boolean,
@@ -969,12 +1070,17 @@ object Tasks {
 
       val verbosityLevel = coursierVerbosity.value
 
-      val res = {
+      val res =
-        if (withClassifiers && sbtClassifiers)
+        if (withClassifiers && sbtClassifiers) {
-          coursierSbtClassifiersResolution
+          val r = coursierSbtClassifiersResolution.value
-        else
+          Map(cm.configurations.map(c => c.name).toSet -> r)
-          coursierResolution
+        } else
-      }.value
+          coursierResolutions.value
+
+      val configResolutions = res.flatMap {
+        case (configs, r) =>
+          configs.iterator.map((_, r))
+      }
 
       def report = {
 
@@ -999,13 +1105,13 @@ object Tasks {
         }
 
         if (verbosityLevel >= 2) {
-          val finalDeps = Config.dependenciesWithConfig(
+          val finalDeps = dependenciesWithConfig(
-            res,
+            configResolutions,
             depsByConfig.map { case (k, l) => k -> l.toSet },
             configs
           )
 
-          val projCache = res.projectCache.mapValues { case (_, p) => p }
+          val projCache = res.values.foldLeft(Map.empty[ModuleVersion, Project])(_ ++ _.projectCache.mapValues(_._2))
           val repr = Print.dependenciesUnknownConfigs(finalDeps.toVector, projCache)
           log.info(repr.split('\n').map("  " + _).mkString("\n"))
         }
@@ -1058,7 +1164,7 @@ object Tasks {
 
         ToSbt.updateReport(
           depsByConfig,
-          res,
+          configResolutions,
           configs,
           classifiers,
           artifactFileOpt(
@@ -1112,37 +1218,43 @@ object Tasks {
         proj.copy(publications = publications)
       }
 
-    val res = {
+    val resolutions =
-      if (sbtClassifiers)
+      if (sbtClassifiers) {
-        coursierSbtClassifiersResolution
+        val r = coursierSbtClassifiersResolution.value
-      else
+        Map(currentProject.configurations.keySet -> r)
-        coursierResolution
+      } else
-    }.value
+        coursierResolutions.value
 
     val config = configuration.value.name
     val configs = coursierConfigurations.value
 
     val includedConfigs = configs.getOrElse(config, Set.empty) + config
 
-    val dependencies0 = currentProject.dependencies.collect {
+    for {
-      case (cfg, dep) if includedConfigs(cfg) => dep
+      (subGraphConfigs, res) <- resolutions
-    }.sortBy { dep =>
+      if subGraphConfigs.exists(includedConfigs)
-      (dep.module.organization, dep.module.name, dep.version)
+    } {
-    }
 
-    val subRes = res.subset(dependencies0.toSet)
+      val dependencies0 = currentProject.dependencies.collect {
+        case (cfg, dep) if includedConfigs(cfg) && subGraphConfigs(cfg) => dep
+      }.sortBy { dep =>
+        (dep.module.organization, dep.module.name, dep.version)
+      }
 
-    // use sbt logging?
+      val subRes = res.subset(dependencies0.toSet)
-    println(
+
-      projectName + "\n" +
+      // use sbt logging?
-      Print.dependencyTree(
+      println(
-        dependencies0,
+        s"$projectName (configurations ${subGraphConfigs.toVector.sorted.mkString(", ")})" + "\n" +
-        subRes,
+          Print.dependencyTree(
-        printExclusions = true,
+            dependencies0,
-        inverse,
+            subRes,
-        colors = !sys.props.get("sbt.log.noformat").toSeq.contains("true")
+            printExclusions = true,
+            inverse,
+            colors = !sys.props.get("sbt.log.noformat").toSeq.contains("true")
+          )
       )
-    )
+    }
   }
 
 }

sbt-coursier/src/main/scala/coursier/ToSbt.scala

@@ -180,7 +180,7 @@ object ToSbt {
 
   def updateReport(
     configDependencies: Map[String, Seq[Dependency]],
-    resolution: Resolution,
+    resolutions: Map[String, Resolution],
     configs: Map[String, Set[String]],
     classifiersOpt: Option[Seq[String]],
     artifactFileOpt: (Module, String, Artifact) => Option[File],
@@ -190,7 +190,7 @@ object ToSbt {
     val configReports = configs.map {
       case (config, extends0) =>
         val configDeps = extends0.flatMap(configDependencies.getOrElse(_, Nil))
-        val subRes = resolution.subset(configDeps)
+        val subRes = resolutions(config).subset(configDeps)
 
         val reports = ToSbt.moduleReports(subRes, classifiersOpt, artifactFileOpt, keepPomArtifact)
 

sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/build.sbt

@@ -0,0 +1,2 @@
+scalaVersion := "2.12.2"
+enablePlugins(ScalafmtPlugin)

sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/project/plugins.sbt

@@ -0,0 +1,13 @@
+{
+  val pluginVersion = sys.props.getOrElse(
+    "plugin.version",
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin
+    )
+  )
+
+  addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
+}
+
+addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "0.3")

sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/src/main/scala/Main.scala

@@ -0,0 +1,6 @@
+import java.io.File
+import java.nio.file.Files
+
+object Main extends App {
+  Files.write(new File("output").toPath, "OK".getBytes("UTF-8"))
+}

sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/test

@@ -0,0 +1 @@
+> scalafmt

sbt-coursier/src/sbt-test/sbt-coursier/credentials-from-file/test

@@ -1 +1 @@
-> coursierResolution
+> coursierResolutions

sbt-coursier/src/sbt-test/sbt-coursier/credentials-sbt/test

@@ -1 +1 @@
-> coursierResolution
+> coursierResolutions

sbt-coursier/src/sbt-test/sbt-coursier/credentials/test

@@ -1 +1 @@
-> coursierResolution
+> coursierResolutions

sbt-shading/src/main/scala/coursier/ShadingPlugin.scala

@@ -102,7 +102,16 @@ object ShadingPlugin extends AutoPlugin {
           toShadeJars := {
             coursier.Shading.toShadeJars(
               coursierProject.in(baseSbtConfiguration).value,
-              coursierResolution.in(baseSbtConfiguration).value,
+              coursierResolutions
+                .in(baseSbtConfiguration)
+                .value
+                .collectFirst {
+                  case (configs, res) if configs(baseDependencyConfiguration) =>
+                    res
+                }
+                .getOrElse {
+                  sys.error(s"Resolution for configuration $baseDependencyConfiguration not found")
+                },
               coursierConfigurations.in(baseSbtConfiguration).value,
               Keys.coursierArtifacts.in(baseSbtConfiguration).value,
               classpathTypes.value,