Run resolution for each configuration

Rather than for each configuration sub-graph. This runs a resolution for
Compile, one for Runtime, one for Test, etc. rather than one for all 3
of them, for example.

We re-use the Resolution instance from the first extended configuration,
so that the performance penalty is really low.
This commit is contained in:
Alexandre Archambault 2020-05-18 12:57:28 +02:00
parent 627877fcc7
commit 3cd521c43c
22 changed files with 171 additions and 76 deletions

View File

@ -129,9 +129,11 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
(ToCoursier.configuration(config), ToCoursier.dependency(dep0))
}
val configGraphs = Inputs.ivyGraphs(
Inputs.configExtends(module0.configurations)
).map(_.map(ToCoursier.configuration))
val orderedConfigs = Inputs.orderedConfigurations(Inputs.configExtendsSeq(module0.configurations))
.map {
case (config, extends0) =>
(ToCoursier.configuration(config), extends0.map(ToCoursier.configuration))
}
val typelevel = so == Typelevel.typelevelOrg
@ -146,7 +148,7 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
val resolutionParams = ResolutionParams(
dependencies = dependencies,
fallbackDependencies = conf.fallbackDependencies,
configGraphs = configGraphs,
orderedConfigs = orderedConfigs,
autoScalaLibOpt = if (conf.autoScalaLibrary) Some((so, sv)) else None,
mainRepositories = mainRepositories,
parentProjectCache = Map.empty,
@ -167,10 +169,10 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
missingOk = conf.missingOk,
)
def artifactsParams(resolutions: Map[Set[Configuration], Resolution]): ArtifactsParams =
def artifactsParams(resolutions: Map[Configuration, Resolution]): ArtifactsParams =
ArtifactsParams(
classifiers = classifiers,
resolutions = resolutions.values.toSeq,
resolutions = resolutions.values.toSeq.distinct,
includeSignatures = false,
loggerOpt = loggerOpt,
projectName = projectName,
@ -193,7 +195,7 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
}
def updateParams(
resolutions: Map[Set[Configuration], Resolution],
resolutions: Map[Configuration, Resolution],
artifacts: Seq[(Dependency, Publication, Artifact, Option[File])]
) =
UpdateParams(

View File

@ -15,6 +15,11 @@ object Inputs {
Configuration(from.value) -> Configuration(to.value)
}
def configExtendsSeq(configurations: Seq[sbt.librarymanagement.Configuration]): Seq[(Configuration, Seq[Configuration])] =
configurations
.map(cfg => Configuration(cfg.name) -> cfg.extendsConfigs.map(c => Configuration(c.name)))
@deprecated("Now unused internally, to be removed in the future", "2.0.0-RC6-5")
def configExtends(configurations: Seq[sbt.librarymanagement.Configuration]): Map[Configuration, Seq[Configuration]] =
configurations
.map(cfg => Configuration(cfg.name) -> cfg.extendsConfigs.map(c => Configuration(c.name)))
@ -25,7 +30,7 @@ object Inputs {
shadedConfig: Option[(String, Configuration)] = None
): Map[Configuration, Set[Configuration]] = {
val configs0 = configExtends(configurations)
val configs0 = configExtendsSeq(configurations).toMap
def allExtends(c: Configuration) = {
// possibly bad complexity
@ -55,6 +60,29 @@ object Inputs {
}
}
def orderedConfigurations(
configurations: Seq[(Configuration, Seq[Configuration])]
): Seq[(Configuration, Seq[Configuration])] = {
val map = configurations.toMap
def helper(done: Set[Configuration], toAdd: List[Configuration]): Stream[(Configuration, Seq[Configuration])] =
toAdd match {
case Nil => Stream.empty
case config :: rest =>
val extends0 = map.getOrElse(config, Nil)
val missingExtends = extends0.filterNot(done)
if (missingExtends.isEmpty)
(config, extends0) #:: helper(done + config, rest)
else
helper(done, missingExtends.toList ::: toAdd)
}
helper(Set.empty, configurations.map(_._1).toList)
.toVector
}
@deprecated("Now unused internally, to be removed in the future", "2.0.0-RC6-5")
def ivyGraphs(configurations: Map[Configuration, Seq[Configuration]]): Seq[Set[Configuration]] = {
// probably bad complexity, but that shouldn't matter given the size of the graphs involved...

View File

@ -10,11 +10,13 @@ import lmcoursier.FallbackDependency
import lmcoursier.definitions.ToCoursier
import coursier.util.Task
import scala.collection.mutable
// private[coursier]
final case class ResolutionParams(
dependencies: Seq[(Configuration, Dependency)],
fallbackDependencies: Seq[FallbackDependency],
configGraphs: Seq[Set[Configuration]],
orderedConfigs: Seq[(Configuration, Seq[Configuration])],
autoScalaLibOpt: Option[(Organization, String)],
mainRepositories: Seq[Repository],
parentProjectCache: ProjectCache,
@ -30,6 +32,18 @@ final case class ResolutionParams(
missingOk: Boolean,
) {
lazy val allConfigExtends: Map[Configuration, Set[Configuration]] = {
val map = new mutable.HashMap[Configuration, Set[Configuration]]
for ((config, extends0) <- orderedConfigs) {
val allExtends = extends0
.iterator
// the else of the getOrElse shouldn't be hit (because of the ordering of the configurations)
.foldLeft(Set(config))((acc, ext) => acc ++ map.getOrElse(ext, Set(ext)))
map += config -> allExtends
}
map.toMap
}
val fallbackDependenciesRepositories =
if (fallbackDependencies.isEmpty)
Nil

View File

@ -9,6 +9,8 @@ import coursier.maven.MavenRepository
import coursier.params.rule.RuleResolution
import sbt.util.Logger
import scala.collection.mutable
// private[coursier]
object ResolutionRun {
@ -16,7 +18,8 @@ object ResolutionRun {
params: ResolutionParams,
verbosityLevel: Int,
log: Logger,
configs: Set[Configuration]
configs: Set[Configuration],
startingResolutionOpt: Option[Resolution]
): Either[coursier.error.ResolutionError, Resolution] = {
val isScalaToolConfig = configs(Configuration("scala-tool"))
@ -80,6 +83,8 @@ object ResolutionRun {
ThreadUtil.withFixedThreadPool(params.parallel) { pool =>
Resolve()
// re-using various caches from a resolution of a configuration we extend
.withInitialResolution(startingResolutionOpt)
.withDependencies(
params.dependencies.collect {
case (config, dep) if configs(config) =>
@ -126,7 +131,7 @@ object ResolutionRun {
params: ResolutionParams,
verbosityLevel: Int,
log: Logger
): Either[coursier.error.ResolutionError, Map[Set[Configuration], Resolution]] = {
): Either[coursier.error.ResolutionError, Map[Configuration, Resolution]] = {
// TODO Warn about possible duplicated modules from source repositories?
@ -141,13 +146,24 @@ object ResolutionRun {
// Downloads are already parallel, no need to parallelize further, anyway.
val resOrError =
Lock.lock.synchronized {
params.configGraphs.foldLeft[Either[coursier.error.ResolutionError, Map[Set[Configuration], Resolution]]](Right(Map())) {
case (acc, config) =>
var map = new mutable.HashMap[Configuration, Resolution]
val either = params.orderedConfigs.foldLeft[Either[coursier.error.ResolutionError, Unit]](Right(())) {
case (acc, (config, extends0)) =>
for {
m <- acc
res <- resolution(params, verbosityLevel, log, config)
} yield m + (config -> res)
_ <- acc
initRes = {
val it = extends0.iterator.flatMap(map.get(_).iterator)
if (it.hasNext) Some(it.next())
else None
}
allExtends = params.allConfigExtends.getOrElse(config, Set.empty)
res <- resolution(params, verbosityLevel, log, allExtends, initRes)
} yield {
map += config -> res
()
}
}
either.map(_ => map.toMap)
}
for (res <- resOrError)
SbtCoursierCache.default.putResolution(params.resolutionKey, res)

View File

@ -12,15 +12,15 @@ class SbtCoursierCache {
import SbtCoursierCache._
private val resolutionsCache = new ConcurrentHashMap[ResolutionKey, Map[Set[Configuration], Resolution]]
private val resolutionsCache = new ConcurrentHashMap[ResolutionKey, Map[Configuration, Resolution]]
// these may actually not need to be cached any more, now that the resolutions
// are cached
private val reportsCache = new ConcurrentHashMap[ReportKey, UpdateReport]
def resolutionOpt(key: ResolutionKey): Option[Map[Set[Configuration], Resolution]] =
def resolutionOpt(key: ResolutionKey): Option[Map[Configuration, Resolution]] =
Option(resolutionsCache.get(key))
def putResolution(key: ResolutionKey, res: Map[Set[Configuration], Resolution]): Unit =
def putResolution(key: ResolutionKey, res: Map[Configuration, Resolution]): Unit =
resolutionsCache.put(key, res)
def reportOpt(key: ReportKey): Option[UpdateReport] =
@ -53,7 +53,7 @@ object SbtCoursierCache {
final case class ReportKey(
dependencies: Seq[(Configuration, Dependency)],
resolution: Map[Set[Configuration], Resolution],
resolution: Map[Configuration, Resolution],
withClassifiers: Boolean,
sbtClassifiers: Boolean,
includeSignatures: Boolean

View File

@ -132,7 +132,6 @@ private[internal] object SbtUpdateReport {
private def moduleReports(
thisModule: (Module, String),
config: Configuration,
res: Resolution,
interProjectDependencies: Seq[Project],
classifiersOpt: Option[Seq[Classifier]],
@ -293,9 +292,8 @@ private[internal] object SbtUpdateReport {
def apply(
thisModule: (Module, String),
configDependencies: Map[Configuration, Seq[Dependency]],
resolutions: Map[Configuration, Resolution],
resolutions: Seq[(Configuration, Resolution)],
interProjectDependencies: Vector[Project],
configs: Map[Configuration, Set[Configuration]],
classifiersOpt: Option[Seq[Classifier]],
artifactFileOpt: (Module, String, Attributes, Artifact) => Option[File],
fullArtifactsOpt: Option[Map[(Dependency, Publication, Artifact), Option[File]]],
@ -306,18 +304,11 @@ private[internal] object SbtUpdateReport {
missingOk: Boolean
): UpdateReport = {
val configReports = configs.map {
case (config, extends0) =>
val configDeps = extends0
.toSeq
.sortBy(_.value)
.flatMap(configDependencies.getOrElse(_, Nil))
.distinct
val subRes = resolutions(config).subset(configDeps)
val configReports = resolutions.map {
case (config, subRes) =>
val reports = moduleReports(
thisModule,
config,
subRes,
interProjectDependencies,
classifiersOpt,

View File

@ -15,7 +15,7 @@ final case class UpdateParams(
configs: Map[Configuration, Set[Configuration]],
dependencies: Seq[(Configuration, Dependency)],
interProjectDependencies: Seq[Project],
res: Map[Set[Configuration], Resolution],
res: Map[Configuration, Resolution],
includeSignatures: Boolean,
sbtBootJarOverrides: Map[(Module, String), File],
classpathOrder: Boolean,

View File

@ -57,11 +57,6 @@ object UpdateRun {
log: Logger
): UpdateReport = Lock.lock.synchronized {
val configResolutions = params.res.flatMap {
case (configs, r) =>
configs.iterator.map((_, r))
}
val depsByConfig = grouped(params.dependencies)(
config =>
params.shadedConfigOpt match {
@ -74,7 +69,7 @@ object UpdateRun {
if (verbosityLevel >= 2) {
val finalDeps = dependenciesWithConfig(
configResolutions,
params.res,
depsByConfig,
params.configs
)
@ -87,9 +82,8 @@ object UpdateRun {
SbtUpdateReport(
params.thisModule,
depsByConfig,
configResolutions,
params.res.toVector.sortBy(_._1.value), // FIXME Order by config topologically?
params.interProjectDependencies.toVector,
params.configs,
params.classifiers,
params.artifactFileOpt,
params.fullArtifacts,

View File

@ -37,7 +37,7 @@ object InputsTasks {
val exclusions0 = Inputs.exclusions(excludeDeps, sv, sbv, log)
val configMap = Inputs.configExtends(configurations)
val configMap = Inputs.configExtendsSeq(configurations).toMap
val proj = FromSbt.project(
projId,

View File

@ -176,13 +176,10 @@ object CoursierPlugin extends AutoPlugin {
coursierResolutions
.value
.collectFirst {
case (configs, res) if configs(config) =>
res
}
.getOrElse {
.getOrElse(
config,
sys.error(s"Resolution for configuration $config not found")
}
)
},
coursierSbtClassifiersResolution := (Def.taskDyn {
val missingOk = (updateConfiguration in updateSbtClassifiers).value.missingOk

View File

@ -13,7 +13,7 @@ import scala.collection.mutable
object DisplayTasks {
private case class ResolutionResult(configs: Set[Configuration], resolution: Resolution, dependencies: Seq[Dependency])
private case class ResolutionResult(config: Configuration, resolution: Resolution, dependencies: Seq[Dependency])
private def coursierResolutionTask(
sbtClassifiers: Boolean = false,
@ -40,7 +40,11 @@ object DisplayTasks {
Def.task {
val currentProject = currentProjectTask.value
val classifiersRes = coursierSbtClassifiersResolution.value
Map(currentProject.configurations.keySet.map(ToCoursier.configuration) -> classifiersRes)
currentProject
.configurations
.keysIterator
.map(config => ToCoursier.configuration(config) -> classifiersRes)
.toMap
}
else
Def.task(coursierResolutions.value)
@ -57,19 +61,23 @@ object DisplayTasks {
val resolutions = resolutionsTask.value
for {
(subGraphConfigs, res) <- resolutions.toSeq
if subGraphConfigs.exists(includedConfigs)
(subGraphConfig, res) <- resolutions.toSeq
if includedConfigs(subGraphConfig)
} yield {
val dependencies0 = currentProject.dependencies.collect {
case (cfg, dep) if includedConfigs(cfg) && subGraphConfigs(cfg) => dep
}.sortBy { dep =>
(dep.module.organization, dep.module.name, dep.version)
}
val dependencies0 = currentProject
.dependencies
.collect {
case (`subGraphConfig`, dep) =>
dep
}
.sortBy { dep =>
(dep.module.organization, dep.module.name, dep.version)
}
val subRes = res.subset(dependencies0)
ResolutionResult(subGraphConfigs, subRes, dependencies0)
ResolutionResult(subGraphConfig, subRes, dependencies0)
}
}
}
@ -82,9 +90,9 @@ object DisplayTasks {
val projectName = thisProjectRef.value.project
val resolutions = coursierResolutionTask(sbtClassifiers, ignoreArtifactErrors).value
for (ResolutionResult(subGraphConfigs, resolution, dependencies) <- resolutions) {
for (ResolutionResult(subGraphConfig, resolution, dependencies) <- resolutions) {
streams.value.log.info(
s"$projectName (configurations ${subGraphConfigs.toVector.sorted.mkString(", ")})" + "\n" +
s"$projectName (configuration ${subGraphConfig.value})" + "\n" +
Print.dependencyTree(
resolution,
dependencies,
@ -110,13 +118,13 @@ object DisplayTasks {
val resolutions = coursierResolutionTask(sbtClassifiers, ignoreArtifactErrors).value
val result = new mutable.StringBuilder
for (ResolutionResult(subGraphConfigs, resolution, _) <- resolutions) {
for (ResolutionResult(subGraphConfig, resolution, _) <- resolutions) {
val roots = resolution
.minDependencies
.filter(f => f.module == module)
.toVector
.sortBy(_.toString) // elements already have the same module, there's not much left for sorting
val strToPrint = s"$projectName (configurations ${subGraphConfigs.toVector.sorted.map(_.value).mkString(", ")})" + "\n" +
val strToPrint = s"$projectName (configurations ${subGraphConfig.value})" + "\n" +
Print.dependencyTree(
resolution,
roots,

View File

@ -26,10 +26,13 @@ object InputsTasks {
}
}
def ivyGraphsTask: Def.Initialize[sbt.Task[Seq[Set[Configuration]]]] =
def ivyGraphsTask: Def.Initialize[sbt.Task[Seq[(Configuration, Seq[Configuration])]]] =
Def.task {
val p = coursierProject.value
Inputs.ivyGraphs(p.configurations).map(_.map(ToCoursier.configuration))
Inputs.orderedConfigurations(p.configurations.toSeq).map {
case (config, extends0) =>
(ToCoursier.configuration(config), extends0.map(ToCoursier.configuration))
}
}
def parentProjectCacheTask: Def.Initialize[sbt.Task[Map[Seq[sbt.librarymanagement.Resolver], Seq[coursier.ProjectCache]]]] =
@ -53,7 +56,7 @@ object InputsTasks {
n.foldLeft(Map.empty[Seq[Resolver], Seq[ProjectCache]]) {
case (caches, (ref, resolutions)) =>
val mainResOpt = resolutions.collectFirst {
case (k, v) if k(Configuration.compile) => v
case (Configuration.compile, v) => v
}
val r = for {

View File

@ -21,7 +21,7 @@ object Keys {
val coursierVerbosity = SettingKey[Int]("coursier-verbosity")
val coursierConfigGraphs = TaskKey[Seq[Set[Configuration]]]("coursier-config-graphs")
val coursierConfigGraphs = TaskKey[Seq[(Configuration, Seq[Configuration])]]("coursier-config-graphs")
val coursierSbtClassifiersModule = TaskKey[GetClassifiersModule]("coursier-sbt-classifiers-module")
@ -29,7 +29,7 @@ object Keys {
val coursierParentProjectCache = TaskKey[Map[Seq[Resolver], Seq[ProjectCache]]]("coursier-parent-project-cache")
val coursierResolutions = TaskKey[Map[Set[Configuration], Resolution]]("coursier-resolutions")
val coursierResolutions = TaskKey[Map[Configuration, Resolution]]("coursier-resolutions")
private[coursier] val actualCoursierResolution = TaskKey[Resolution]("coursier-resolution")

View File

@ -19,9 +19,9 @@ object ResolutionTasks {
def resolutionsTask(
sbtClassifiers: Boolean = false,
missingOk: Boolean = false,
): Def.Initialize[sbt.Task[Map[Set[Configuration], coursier.Resolution]]] = {
): Def.Initialize[sbt.Task[Map[Configuration, coursier.Resolution]]] = {
val currentProjectTask: sbt.Def.Initialize[sbt.Task[(Project, Seq[FallbackDependency], Seq[Set[Configuration]])]] =
val currentProjectTask: sbt.Def.Initialize[sbt.Task[(Project, Seq[FallbackDependency], Seq[(Configuration, Seq[Configuration])])]] =
if (sbtClassifiers)
Def.task {
val sv = scalaVersion.value
@ -35,7 +35,7 @@ object ResolutionTasks {
sbv
)
(proj, fallbackDeps, Vector(cm.configurations.map(c => Configuration(c.name)).toSet))
(proj, fallbackDeps, cm.configurations.map(c => Configuration(c.name) -> Nil))
}
else
Def.task {
@ -97,7 +97,7 @@ object ResolutionTasks {
val authenticationByRepositoryId = coursierCredentials.value.mapValues(_.authentication)
val (currentProject, fallbackDependencies, configGraphs) = currentProjectTask.value
val (currentProject, fallbackDependencies, orderedConfigs) = currentProjectTask.value
val autoScalaLib = autoScalaLibrary.value && scalaModuleInfo.value.forall(_.overrideScalaVersion)
@ -132,7 +132,7 @@ object ResolutionTasks {
ResolutionParams(
dependencies = currentProject.dependencies,
fallbackDependencies = fallbackDependencies,
configGraphs = configGraphs,
orderedConfigs = orderedConfigs,
autoScalaLibOpt = if (autoScalaLib) Some((so, sv)) else None,
mainRepositories = mainRepositories,
parentProjectCache = parentProjectCache,

View File

@ -35,9 +35,13 @@ object UpdateTasks {
val resTask =
if (withClassifiers && sbtClassifiers)
Def.task {
val cm = coursierSbtClassifiersModule.value
val mod = coursierSbtClassifiersModule.value
val classifiersRes = coursierSbtClassifiersResolution.value
Map(cm.configurations.map(c => Configuration(c.name)).toSet -> classifiersRes)
mod
.configurations
.iterator
.map(c => Configuration(c.name) -> classifiersRes)
.toMap
}
else
Def.task(coursierResolutions.value)

View File

@ -12,5 +12,6 @@ import CoursierPlugin.autoImport._
whatDependsOnCheck := {
val result = (coursierWhatDependsOn in Compile).toTask(" log4j:log4j").value
val file = new File("whatDependsOnResult.log")
assert(IO.read(file).toString == result)
val expected = IO.read(file).toString
assert(expected == result, s"Expected '$expected', got '$result'")
}

View File

@ -1,4 +1,4 @@
dependency-graph (configurations compile, compile-internal, optional, provided, runtime, runtime-internal, test, test-internal)
dependency-graph (configurations compile)
└─ log4j:log4j:1.2.17
├─ org.apache.zookeeper:zookeeper:3.5.0-alpha log4j:log4j:1.2.16 -> 1.2.17
└─ org.slf4j:slf4j-log4j12:1.7.5

View File

@ -0,0 +1,8 @@
scalaVersion := "2.13.2"
libraryDependencies ++= Seq(
"io.get-coursier" %% "coursier-core" % "2.0.0-RC6",
// depends on coursier-core 2.0.0-RC6-16
"io.get-coursier" %% "coursier" % "2.0.0-RC6-16" % Test
)
mainClass.in(Compile) := Some("Main")
mainClass.in(Test) := Some("Test")

View File

@ -0,0 +1,13 @@
addSbtPlugin {
val name = sys.props.getOrElse(
"plugin.name",
sys.error("plugin.name Java property not set")
)
val version = sys.props.getOrElse(
"plugin.version",
sys.error("plugin.version Java property not set")
)
"io.get-coursier" % name % version
}

View File

@ -0,0 +1,7 @@
object Main {
def main(args: Array[String]): Unit = {
val version = coursier.util.Properties.version
val expected = "2.0.0-RC6"
assert(version == expected, s"version: $version, expected: $expected")
}
}

View File

@ -0,0 +1,7 @@
object Test {
def main(args: Array[String]): Unit = {
val version = coursier.util.Properties.version
val expected = "2.0.0-RC6-16"
assert(version == expected, s"version: $version, expected: $expected")
}
}

View File

@ -0,0 +1,2 @@
> run
> test:run