luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Run separate resolutions for distinct Ivy configurations subgraphs

This commit is contained in:
Alexandre Archambault 2017-05-30 18:41:21 +02:00
parent 18be2e2dd2
commit 8002910a43
12 changed files with 255 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
sbt-coursier/src/main/scala/coursier/CoursierPlugin.scala
View File

@ -28,6 +28,7 @@ object CoursierPlugin extends AutoPlugin {
val coursierFallbackDependencies = Keys.coursierFallbackDependencies
val coursierCache = Keys.coursierCache
val coursierProject = Keys.coursierProject
val coursierConfigGraphs = Keys.coursierConfigGraphs
val coursierInterProjectDependencies = Keys.coursierInterProjectDependencies
val coursierPublications = Keys.coursierPublications
val coursierSbtClassifiersModule = Keys.coursierSbtClassifiersModule
@ -35,7 +36,11 @@ object CoursierPlugin extends AutoPlugin {
val coursierConfigurations = Keys.coursierConfigurations
val coursierParentProjectCache = Keys.coursierParentProjectCache
val coursierResolution = Keys.coursierResolution
val coursierResolutions = Keys.coursierResolutions
@deprecated("Use coursierResolutions instead", "1.0.0-RC4")
val coursierResolution = Keys.actualCoursierResolution
val coursierSbtClassifiersResolution = Keys.coursierSbtClassifiersResolution
val coursierDependencyTree = Keys.coursierDependencyTree
@ -117,15 +122,30 @@ object CoursierPlugin extends AutoPlugin {
ignoreArtifactErrors = true
).value,
coursierProject := Tasks.coursierProjectTask.value,
coursierConfigGraphs := Tasks.ivyGraphsTask.value,
coursierInterProjectDependencies := Tasks.coursierInterProjectDependenciesTask.value,
coursierPublications := Tasks.coursierPublicationsTask(packageConfigs: _*).value,
coursierSbtClassifiersModule := classifiersModule.in(updateSbtClassifiers).value,
coursierConfigurations := Tasks.coursierConfigurationsTask(None).value,
coursierParentProjectCache := Tasks.parentProjectCacheTask.value,
coursierResolution := Tasks.resolutionTask().value,
coursierSbtClassifiersResolution := Tasks.resolutionTask(
coursierResolutions := Tasks.resolutionsTask().value,
Keys.actualCoursierResolution := {
val config = Compile.name
coursierResolutions
.value
.collectFirst {
case (configs, res) if (configs(config)) =>
res
}
.getOrElse {
sys.error(s"Resolution for configuration $config not found")
}
},
coursierSbtClassifiersResolution := Tasks.resolutionsTask(
sbtClassifiers = true
).value,
).value.head._2,
ivyConfigurations := {
val confs = ivyConfigurations.value
val names = confs.map(_.name).toSet

8
sbt-coursier/src/main/scala/coursier/Keys.scala
View File

@ -34,6 +34,7 @@ object Keys {
val coursierFallbackDependencies = TaskKey[Seq[(Module, String, URL, Boolean)]]("coursier-fallback-dependencies")
val coursierProject = TaskKey[Project]("coursier-project")
val coursierConfigGraphs = TaskKey[Seq[Set[String]]]("coursier-config-graphs")
val coursierInterProjectDependencies = TaskKey[Seq[Project]]("coursier-inter-project-dependencies", "Projects the current project depends on, possibly transitively")
val coursierPublications = TaskKey[Seq[(String, Publication)]]("coursier-publications")
@ -43,7 +44,12 @@ object Keys {
val coursierParentProjectCache = TaskKey[Map[Seq[Resolver], Seq[ProjectCache]]]("coursier-parent-project-cache")
val coursierResolution = TaskKey[Resolution]("coursier-resolution")
val coursierResolutions = TaskKey[Map[Set[String], Resolution]]("coursier-resolutions")
private[coursier] val actualCoursierResolution = TaskKey[Resolution]("coursier-resolution")
@deprecated("Use coursierResolutions instead", "1.0.0-RC4")
val coursierResolution = actualCoursierResolution
val coursierSbtClassifiersResolution = TaskKey[Resolution]("coursier-sbt-classifiers-resolution")
val coursierDependencyTree = TaskKey[Unit](

262
sbt-coursier/src/main/scala/coursier/Tasks.scala
View File

@ -9,7 +9,7 @@ import coursier.extra.Typelevel
import coursier.ivy.{IvyRepository, PropertiesPattern}
import coursier.Keys._
import coursier.Structure._
import coursier.util.{Config, Print}
import coursier.util.Print
import sbt.{Classpaths, Def, Resolver, UpdateReport}
import sbt.Keys._
@ -344,18 +344,18 @@ object Tasks {
project: Project,
repositories: Seq[Repository],
userEnabledProfiles: Set[String],
resolution: Resolution,
resolution: Map[Set[String], Resolution],
sbtClassifiers: Boolean
)
private final case class ReportCacheKey(
project: Project,
resolution: Resolution,
resolution: Map[Set[String], Resolution],
withClassifiers: Boolean,
sbtClassifiers: Boolean
)
private val resolutionsCache = new mutable.HashMap[ResolutionCacheKey, Resolution]
private val resolutionsCache = new mutable.HashMap[ResolutionCacheKey, Map[Set[String], Resolution]]
// these may actually not need to be cached any more, now that the resolutions
// are cached
private val reportsCache = new mutable.HashMap[ReportCacheKey, UpdateReport]
@ -423,28 +423,76 @@ object Tasks {
val t =
for {
m <- coursierRecursiveResolvers.forAllProjects(state, projects)
n <- coursierResolution.forAllProjects(state, m.keys.toSeq)
n <- coursierResolutions.forAllProjects(state, m.keys.toSeq)
} yield
n.foldLeft(Map.empty[Seq[Resolver], Seq[ProjectCache]]) {
case (caches, (ref, resolution)) =>
m.get(ref).fold(caches) { resolvers =>
caches.updated(
resolvers,
resolution.projectCache +: caches.getOrElse(resolvers, Nil)
)
case (caches, (ref, resolutions)) =>
val mainResOpt = resolutions.collectFirst {
case (k, v) if k("compile") => v
}
val r = for {
resolvers <- m.get(ref)
resolution <- mainResOpt
} yield
caches.updated(resolvers, resolution.projectCache +: caches.getOrElse(resolvers, Seq.empty))
r.getOrElse(caches)
}
Def.task(t.value)
}
def ivyGraphsTask = Def.task {
// probably bad complexity, but that shouldn't matter given the size of the graphs involved...
val p = coursierProject.value
final class Wrapper(val set: mutable.HashSet[String]) {
def ++=(other: Wrapper): this.type = {
set ++= other.set
this
}
}
val sets =
new mutable.HashMap[String, Wrapper] ++= p.configurations.map {
case (k, l) =>
val s = new mutable.HashSet[String]()
s ++= l
s += k
k -> new Wrapper(s)
}
for (k <- p.configurations.keys) {
val s = sets(k)
var foundNew = true
while (foundNew) {
foundNew = false
for (other <- s.set.toVector) {
val otherS = sets(other)
if (!otherS.eq(s)) {
s ++= otherS
sets += other -> s
foundNew = true
}
}
}
}
sets.values.toVector.distinct.map(_.set.toSet)
}
private val noOptionalFilter: Option[Dependency => Boolean] = Some(dep => !dep.optional)
private val typelevelOrgSwap: Option[Dependency => Dependency] = Some(Typelevel.swap(_))
def resolutionTask(
def resolutionsTask(
sbtClassifiers: Boolean = false
): Def.Initialize[sbt.Task[coursier.Resolution]] = Def.task {
): Def.Initialize[sbt.Task[Map[Set[String], coursier.Resolution]]] = Def.task {
// let's update only one module at once, for a better output
// Downloads are already parallel, no need to parallelize further anyway
@ -454,7 +502,7 @@ object Tasks {
lazy val projectName = thisProjectRef.value.project
val (currentProject, fallbackDependencies) =
val (currentProject, fallbackDependencies, configGraphs) =
if (sbtClassifiers) {
val sv = scalaVersion.value
val sbv = scalaBinaryVersion.value
@ -473,12 +521,12 @@ object Tasks {
sbv
)
(proj, fallbackDeps)
(proj, fallbackDeps, Vector(cm.configurations.map(_.name).toSet))
} else {
val proj = coursierProject.value
val publications = coursierPublications.value
val fallbackDeps = coursierFallbackDependencies.value
(proj.copy(publications = publications), fallbackDeps)
(proj.copy(publications = publications), fallbackDeps, coursierConfigGraphs.value)
}
val interProjectDependencies = coursierInterProjectDependencies.value
@ -520,23 +568,6 @@ object Tasks {
val typelevel = scalaOrganization.value == Typelevel.typelevelOrg
val startRes = Resolution(
currentProject.dependencies.map(_._2).toSet,
filter = noOptionalFilter,
userActivations =
if (userEnabledProfiles.isEmpty)
None
else
Some(userEnabledProfiles.iterator.map(_ -> true).toMap),
forceVersions =
// order matters here
userForceVersions ++
forcedScalaModules(so, sv) ++
interProjectDependencies.map(_.moduleVersion),
projectCache = parentProjectCache,
mapDependencies = if (typelevel) typelevelOrgSwap else None
)
if (verbosityLevel >= 2) {
log.info("InterProjectRepository")
for (p <- interProjectDependencies)
@ -653,7 +684,31 @@ object Tasks {
}.map(withAuthenticationByHost(_, authenticationByHost)) ++
fallbackDependenciesRepositories
def resolution = {
def startRes(configs: Set[String]) = Resolution(
currentProject
.dependencies
.collect {
case (config, dep) if configs(config) =>
dep
}
.toSet,
filter = noOptionalFilter,
userActivations =
if (userEnabledProfiles.isEmpty)
None
else
Some(userEnabledProfiles.iterator.map(_ -> true).toMap),
forceVersions =
// order matters here
userForceVersions ++
(if (configs("compile") || configs("scala-tool")) forcedScalaModules(so, sv) else Map()) ++
interProjectDependencies.map(_.moduleVersion),
projectCache = parentProjectCache,
mapDependencies = if (typelevel && (configs("compile") || configs("scala-tool"))) typelevelOrgSwap else None
)
def resolution(startRes: Resolution) = {
var pool: ExecutorService = null
var resLogger: TermDisplay = null
@ -763,15 +818,20 @@ object Tasks {
res
}
val allStartRes = configGraphs.map(configs => configs -> startRes(configs)).toMap
resolutionsCache.getOrElseUpdate(
ResolutionCacheKey(
currentProject,
repositories,
userEnabledProfiles,
startRes.copy(filter = None),
allStartRes,
sbtClassifiers
),
resolution
allStartRes.map {
case (config, startRes) =>
config -> resolution(startRes)
}
)
}
}
@ -800,12 +860,11 @@ object Tasks {
val verbosityLevel = coursierVerbosity.value
val res = {
val res =
if (withClassifiers && sbtClassifiers)
coursierSbtClassifiersResolution
Seq(coursierSbtClassifiersResolution.value)
else
coursierResolution
}.value
coursierResolutions.value.values.toVector
val classifiers =
if (withClassifiers)
@ -820,8 +879,8 @@ object Tasks {
val allArtifacts =
classifiers match {
case None => res.artifacts
case Some(cl) => res.classifiersArtifacts(cl)
case None => res.flatMap(_.artifacts)
case Some(cl) => res.flatMap(_.classifiersArtifacts(cl))
}
var pool: ExecutorService = null
@ -833,7 +892,7 @@ object Tasks {
pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory)
artifactsLogger = createLogger()
val artifactFileOrErrorTasks = allArtifacts.toVector.map { a =>
val artifactFileOrErrorTasks = allArtifacts.toVector.distinct.map { a =>
def f(p: CachePolicy) =
Cache.file(
a,
@ -916,6 +975,48 @@ object Tasks {
res
}
// Move back to coursier.util (in core module) after 1.0?
private def allDependenciesByConfig(
res: Map[String, Resolution],
depsByConfig: Map[String, Set[Dependency]],
configs: Map[String, Set[String]]
): Map[String, Set[Dependency]] = {
val allDepsByConfig = depsByConfig.map {
case (config, deps) =>
config -> res(config).subset(deps).minDependencies
}
val filteredAllDepsByConfig = allDepsByConfig.map {
case (config, allDeps) =>
val allExtendedConfigs = configs.getOrElse(config, Set.empty) - config
val inherited = allExtendedConfigs
.flatMap(allDepsByConfig.getOrElse(_, Set.empty))
config -> (allDeps -- inherited)
}
filteredAllDepsByConfig
}
// Move back to coursier.util (in core module) after 1.0?
private def dependenciesWithConfig(
res: Map[String, Resolution],
depsByConfig: Map[String, Set[Dependency]],
configs: Map[String, Set[String]]
): Set[Dependency] =
allDependenciesByConfig(res, depsByConfig, configs)
.flatMap {
case (config, deps) =>
deps.map(dep => dep.copy(configuration = s"$config->${dep.configuration}"))
}
.groupBy(_.copy(configuration = ""))
.map {
case (dep, l) =>
dep.copy(configuration = l.map(_.configuration).mkString(";"))
}
.toSet
def updateTask(
shadedConfigOpt: Option[(String, String)],
withClassifiers: Boolean,
@ -969,12 +1070,17 @@ object Tasks {
val verbosityLevel = coursierVerbosity.value
val res = {
if (withClassifiers && sbtClassifiers)
coursierSbtClassifiersResolution
else
coursierResolution
}.value
val res =
if (withClassifiers && sbtClassifiers) {
val r = coursierSbtClassifiersResolution.value
Map(cm.configurations.map(c => c.name).toSet -> r)
} else
coursierResolutions.value
val configResolutions = res.flatMap {
case (configs, r) =>
configs.iterator.map((_, r))
}
def report = {
@ -999,13 +1105,13 @@ object Tasks {
}
if (verbosityLevel >= 2) {
val finalDeps = Config.dependenciesWithConfig(
res,
val finalDeps = dependenciesWithConfig(
configResolutions,
depsByConfig.map { case (k, l) => k -> l.toSet },
configs
)
val projCache = res.projectCache.mapValues { case (_, p) => p }
val projCache = res.values.foldLeft(Map.empty[ModuleVersion, Project])(_ ++ _.projectCache.mapValues(_._2))
val repr = Print.dependenciesUnknownConfigs(finalDeps.toVector, projCache)
log.info(repr.split('\n').map(" " + _).mkString("\n"))
}
@ -1058,7 +1164,7 @@ object Tasks {
ToSbt.updateReport(
depsByConfig,
res,
configResolutions,
configs,
classifiers,
artifactFileOpt(
@ -1112,37 +1218,43 @@ object Tasks {
proj.copy(publications = publications)
}
val res = {
if (sbtClassifiers)
coursierSbtClassifiersResolution
else
coursierResolution
}.value
val resolutions =
if (sbtClassifiers) {
val r = coursierSbtClassifiersResolution.value
Map(currentProject.configurations.keySet -> r)
} else
coursierResolutions.value
val config = configuration.value.name
val configs = coursierConfigurations.value
val includedConfigs = configs.getOrElse(config, Set.empty) + config
val dependencies0 = currentProject.dependencies.collect {
case (cfg, dep) if includedConfigs(cfg) => dep
}.sortBy { dep =>
(dep.module.organization, dep.module.name, dep.version)
}
for {
(subGraphConfigs, res) <- resolutions
if subGraphConfigs.exists(includedConfigs)
} {
val subRes = res.subset(dependencies0.toSet)
val dependencies0 = currentProject.dependencies.collect {
case (cfg, dep) if includedConfigs(cfg) && subGraphConfigs(cfg) => dep
}.sortBy { dep =>
(dep.module.organization, dep.module.name, dep.version)
}
// use sbt logging?
println(
projectName + "\n" +
Print.dependencyTree(
dependencies0,
subRes,
printExclusions = true,
inverse,
colors = !sys.props.get("sbt.log.noformat").toSeq.contains("true")
val subRes = res.subset(dependencies0.toSet)
// use sbt logging?
println(
s"$projectName (configurations ${subGraphConfigs.toVector.sorted.mkString(", ")})" + "\n" +
Print.dependencyTree(
dependencies0,
subRes,
printExclusions = true,
inverse,
colors = !sys.props.get("sbt.log.noformat").toSeq.contains("true")
)
)
)
}
}
}

4
sbt-coursier/src/main/scala/coursier/ToSbt.scala
View File

@ -180,7 +180,7 @@ object ToSbt {
def updateReport(
configDependencies: Map[String, Seq[Dependency]],
resolution: Resolution,
resolutions: Map[String, Resolution],
configs: Map[String, Set[String]],
classifiersOpt: Option[Seq[String]],
artifactFileOpt: (Module, String, Artifact) => Option[File],
@ -190,7 +190,7 @@ object ToSbt {
val configReports = configs.map {
case (config, extends0) =>
val configDeps = extends0.flatMap(configDependencies.getOrElse(_, Nil))
val subRes = resolution.subset(configDeps)
val subRes = resolutions(config).subset(configDeps)
val reports = ToSbt.moduleReports(subRes, classifiersOpt, artifactFileOpt, keepPomArtifact)

2
sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/build.sbt Normal file
View File

@ -0,0 +1,2 @@
scalaVersion := "2.12.2"
enablePlugins(ScalafmtPlugin)

13
sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/project/plugins.sbt Normal file
View File

@ -0,0 +1,13 @@
{
val pluginVersion = sys.props.getOrElse(
"plugin.version",
throw new RuntimeException(
"""|The system property 'plugin.version' is not defined.
|Specify this property using the scriptedLaunchOpts -D.""".stripMargin
)
)
addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
}
addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "0.3")

6
sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/src/main/scala/Main.scala Normal file
View File

@ -0,0 +1,6 @@
import java.io.File
import java.nio.file.Files
object Main extends App {
Files.write(new File("output").toPath, "OK".getBytes("UTF-8"))
}

1
sbt-coursier/src/sbt-test/sbt-coursier-0.13/neo-sbt-scalafmt/test Normal file
View File

@ -0,0 +1 @@
> scalafmt

2
sbt-coursier/src/sbt-test/sbt-coursier/credentials-from-file/test
View File

@ -1 +1 @@
> coursierResolution
> coursierResolutions

2
sbt-coursier/src/sbt-test/sbt-coursier/credentials-sbt/test
View File

@ -1 +1 @@
> coursierResolution
> coursierResolutions

2
sbt-coursier/src/sbt-test/sbt-coursier/credentials/test
View File

@ -1 +1 @@
> coursierResolution
> coursierResolutions

11
sbt-shading/src/main/scala/coursier/ShadingPlugin.scala
View File

@ -102,7 +102,16 @@ object ShadingPlugin extends AutoPlugin {
toShadeJars := {
coursier.Shading.toShadeJars(
coursierProject.in(baseSbtConfiguration).value,
coursierResolution.in(baseSbtConfiguration).value,
coursierResolutions
.in(baseSbtConfiguration)
.value
.collectFirst {
case (configs, res) if configs(baseDependencyConfiguration) =>
res
}
.getOrElse {
sys.error(s"Resolution for configuration $baseDependencyConfiguration not found")
},
coursierConfigurations.in(baseSbtConfiguration).value,
Keys.coursierArtifacts.in(baseSbtConfiguration).value,
classpathTypes.value,