Small refactoring

This commit is contained in:
Alexandre Archambault 2015-06-25 00:18:49 +01:00
parent 54338f7b04
commit 0de5330351
3 changed files with 293 additions and 293 deletions

View File

@ -7,7 +7,7 @@ import scala.collection.mutable
import scalaz.concurrent.Task
import scalaz.{EitherT, \/-, \/, -\/}
object Resolver {
object Resolution {
type ModuleVersion = (Module, String)
@ -283,291 +283,6 @@ object Resolver {
}
}
/**
* State of a dependency resolution.
*
* Done if method `isDone` returns `true`.
*
* @param dependencies: current set of dependencies
* @param conflicts: conflicting dependencies
* @param projectsCache: cache of known projects
* @param errors: keeps track of the modules whose project definition could not be found
*/
case class Resolution(rootDependencies: Set[Dependency],
dependencies: Set[Dependency],
conflicts: Set[Dependency],
projectsCache: Map[ModuleVersion, (Repository, Project)],
errors: Map[ModuleVersion, Seq[String]],
filter: Option[Dependency => Boolean],
profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) {
private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]()
private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized {
finalDependenciesCache.getOrElseUpdate(dep,
projectsCache.get(dep.moduleVersion) match {
case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter)
case None => Nil
}
)
}
/**
* Transitive dependencies of the current dependencies, according to what there currently is in cache.
* No attempt is made to solve version conflicts here.
*/
def transitiveDependencies: Seq[Dependency] =
for {
dep <- (dependencies -- conflicts).toList
trDep <- finalDependencies0(dep)
} yield trDep
/**
* The "next" dependency set, made of the current dependencies and their transitive dependencies,
* trying to solve version conflicts. Transitive dependencies are calculated with the current cache.
*
* May contain dependencies added in previous iterations, but no more required. These are filtered below, see
* `newDependencies`.
*
* Returns a tuple made of the conflicting dependencies, and all the dependencies.
*/
def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = {
merge(dependencies ++ transitiveDependencies)
}
/**
* The modules we miss some info about.
*/
def missingFromCache: Set[ModuleVersion] = {
val modules = dependencies.map(_.moduleVersion)
val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion)
(modules ++ nextModules)
.filterNot(mod => projectsCache.contains(mod) || errors.contains(mod))
}
/**
* Whether the resolution is done.
*/
def isDone: Boolean = {
def isFixPoint = {
val (nextConflicts, _) = nextDependenciesAndConflicts
dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet
}
missingFromCache.isEmpty && isFixPoint
}
private def eraseVersion(dep: Dependency) = dep.copy(version = "")
/**
* Returns a map giving the dependencies that brought each of the dependency of the "next" dependency set.
*
* The versions of all the dependencies returned are erased (emptied).
*/
def reverseDependencies: Map[Dependency, Vector[Dependency]] = {
val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts
val trDepsSeq =
for {
dep <- updatedDeps
trDep <- finalDependencies0(dep)
} yield eraseVersion(trDep) -> eraseVersion(dep)
val knownDeps = (updatedDeps ++ updatedConflicts).map(eraseVersion).toSet
trDepsSeq
.groupBy(_._1)
.mapValues(_.map(_._2).toVector)
.filterKeys(knownDeps)
.toList.toMap // Eagerly evaluate filterKeys/mapValues
}
/**
* Returns dependencies from the "next" dependency set, filtering out
* those that are no more required.
*
* The versions of all the dependencies returned are erased (emptied).
*/
def remainingDependencies: Set[Dependency] = {
val rootDependencies0 = rootDependencies.map(eraseVersion)
@tailrec
def helper(reverseDeps: Map[Dependency, Vector[Dependency]]): Map[Dependency, Vector[Dependency]] = {
val (toRemove, remaining) = reverseDeps.partition(kv => kv._2.isEmpty && !rootDependencies0(kv._1))
if (toRemove.isEmpty) reverseDeps
else helper(remaining.mapValues(_.filter(x => remaining.contains(x) || rootDependencies0(x))).toList.toMap)
}
val filteredReverseDependencies = helper(reverseDependencies)
rootDependencies0 ++ filteredReverseDependencies.keys
}
/**
* The final next dependency set, stripped of no more required ones.
*/
def newDependencies: Set[Dependency] = {
val remainingDependencies0 = remainingDependencies
nextDependenciesAndConflicts._2
.filter(dep => remainingDependencies0(eraseVersion(dep)))
.toSet
}
private def nextNoMissingUnsafe: Resolution = {
val (newConflicts, _) = nextDependenciesAndConflicts
copy(dependencies = newDependencies ++ newConflicts, conflicts = newConflicts.toSet)
}
/**
* If no module info is missing, the next state of the resolution, which can be immediately calculated.
* Else, the current resolution itself.
*/
def nextIfNoMissing: Resolution = {
val missing = missingFromCache
if (missing.isEmpty) nextNoMissingUnsafe
else this
}
/**
* Do a new iteration, fetching the missing modules along the way.
*/
def next(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = {
val missing = missingFromCache
if (missing.isEmpty) Task.now(nextNoMissingUnsafe)
else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing)
}
/**
* Required modules for the dependency management of `project`.
*/
def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = {
val approxProperties =
project.parent
.flatMap(projectsCache.get)
.map(_._2.properties)
.fold(project.properties)(mergeProperties(project.properties, _))
val profileDependencies =
profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation)
.flatMap(_.dependencies)
val modules =
(project.dependencies ++ profileDependencies)
.collect{ case dep if dep.scope == Scope.Import => dep.moduleVersion } ++
project.parent
modules.toSet
}
/**
* Missing modules in cache, to get the full list of dependencies of `project`, taking
* dependency management / inheritance into account.
*
* Note that adding the missing modules to the cache may unveil other missing modules, so
* these modules should be added to the cache, and `dependencyManagementMissing` checked again
* for new missing modules.
*/
def dependencyManagementMissing(project: Project): Set[ModuleVersion] = {
@tailrec
def helper(toCheck: Set[ModuleVersion],
done: Set[ModuleVersion],
missing: Set[ModuleVersion]): Set[ModuleVersion] = {
if (toCheck.isEmpty) missing
else if (toCheck.exists(done)) helper(toCheck -- done, done, missing)
else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing)
else if (toCheck.exists(projectsCache.contains)) {
val (checking, remaining) = toCheck.partition(projectsCache.contains)
val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectsCache(mod)._2))
helper(remaining ++ directRequirements, done ++ checking, missing)
} else if (toCheck.exists(errors.contains)) {
val (errored, remaining) = toCheck.partition(errors.contains)
helper(remaining, done ++ errored, missing)
} else
helper(Set.empty, done, missing ++ toCheck)
}
helper(dependencyManagementRequirements(project), Set(project.moduleVersion), Set.empty)
}
/**
* Add dependency management / inheritance related items to `project`, from what's available in cache.
* It is recommended to have fetched what `dependencyManagementMissing` returned prior to calling
* `withDependencyManagement`.
*/
def withDependencyManagement(project: Project): Project = {
val approxProperties =
project.parent
.filter(projectsCache.contains)
.map(projectsCache(_)._2.properties)
.fold(project.properties)(mergeProperties(project.properties, _))
val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation)
val dependencies0 = addDependencies(project.dependencies +: profiles0.map(_.dependencies))
val properties0 = (project.properties /: profiles0)((acc, p) => mergeProperties(acc, p.properties))
val deps =
dependencies0
.collect{ case dep if dep.scope == Scope.Import && projectsCache.contains(dep.moduleVersion) => dep.moduleVersion } ++
project.parent.filter(projectsCache.contains)
val projs = deps.map(projectsCache(_)._2)
val depMgmt =
(project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement)))
.foldLeft(Map.empty[DepMgmtKey, Dependency])(dependencyManagementAddSeq)
val depsSet = deps.toSet
project.copy(
dependencies = dependencies0
.filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++
project.parent
.filter(projectsCache.contains)
.toSeq
.flatMap(projectsCache(_)._2.dependencies),
dependencyManagement = depMgmt.values.toSeq,
properties = project.parent
.filter(projectsCache.contains)
.map(projectsCache(_)._2.properties)
.fold(properties0)(mergeProperties(properties0, _))
)
}
/**
* Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache.
*/
def fetch(modules: Seq[ModuleVersion],
fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = {
val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _))
val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true)
gatheredLookups.flatMap{ lookupResults =>
val errors0 = errors ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors}
val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj}
/*
* newProjects are project definitions, fresh from the repositories. We need to add
* dependency management / inheritance-related bits to them.
*/
newProjects.foldLeft(Task.now(copy(errors = errors0))) { case (accTask, (modVer, (repo, proj))) =>
for {
current <- accTask
updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule)
proj0 = updated.withDependencyManagement(proj)
} yield updated.copy(projectsCache = updated.projectsCache + (modVer -> (repo, proj0)))
}
}
}
}
/**
* Default function checking whether a profile is active, given its id, activation conditions,
* and the properties of its project.
@ -630,5 +345,291 @@ object Resolver {
helper(startResolution, maxIterations).map(_._1)
}
}
/**
* State of a dependency resolution.
*
* Done if method `isDone` returns `true`.
*
* @param dependencies: current set of dependencies
* @param conflicts: conflicting dependencies
* @param projectsCache: cache of known projects
* @param errors: keeps track of the modules whose project definition could not be found
*/
case class Resolution(rootDependencies: Set[Dependency],
dependencies: Set[Dependency],
conflicts: Set[Dependency],
projectsCache: Map[Resolution.ModuleVersion, (Repository, Project)],
errors: Map[Resolution.ModuleVersion, Seq[String]],
filter: Option[Dependency => Boolean],
profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) {
import Resolution._
private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]()
private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized {
finalDependenciesCache.getOrElseUpdate(dep,
projectsCache.get(dep.moduleVersion) match {
case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter)
case None => Nil
}
)
}
/**
* Transitive dependencies of the current dependencies, according to what there currently is in cache.
* No attempt is made to solve version conflicts here.
*/
def transitiveDependencies: Seq[Dependency] =
for {
dep <- (dependencies -- conflicts).toList
trDep <- finalDependencies0(dep)
} yield trDep
/**
* The "next" dependency set, made of the current dependencies and their transitive dependencies,
* trying to solve version conflicts. Transitive dependencies are calculated with the current cache.
*
* May contain dependencies added in previous iterations, but no more required. These are filtered below, see
* `newDependencies`.
*
* Returns a tuple made of the conflicting dependencies, and all the dependencies.
*/
def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = {
merge(dependencies ++ transitiveDependencies)
}
/**
* The modules we miss some info about.
*/
def missingFromCache: Set[ModuleVersion] = {
val modules = dependencies.map(_.moduleVersion)
val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion)
(modules ++ nextModules)
.filterNot(mod => projectsCache.contains(mod) || errors.contains(mod))
}
/**
* Whether the resolution is done.
*/
def isDone: Boolean = {
def isFixPoint = {
val (nextConflicts, _) = nextDependenciesAndConflicts
dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet
}
missingFromCache.isEmpty && isFixPoint
}
private def eraseVersion(dep: Dependency) = dep.copy(version = "")
/**
* Returns a map giving the dependencies that brought each of the dependency of the "next" dependency set.
*
* The versions of all the dependencies returned are erased (emptied).
*/
def reverseDependencies: Map[Dependency, Vector[Dependency]] = {
val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts
val trDepsSeq =
for {
dep <- updatedDeps
trDep <- finalDependencies0(dep)
} yield eraseVersion(trDep) -> eraseVersion(dep)
val knownDeps = (updatedDeps ++ updatedConflicts).map(eraseVersion).toSet
trDepsSeq
.groupBy(_._1)
.mapValues(_.map(_._2).toVector)
.filterKeys(knownDeps)
.toList.toMap // Eagerly evaluate filterKeys/mapValues
}
/**
* Returns dependencies from the "next" dependency set, filtering out
* those that are no more required.
*
* The versions of all the dependencies returned are erased (emptied).
*/
def remainingDependencies: Set[Dependency] = {
val rootDependencies0 = rootDependencies.map(eraseVersion)
@tailrec
def helper(reverseDeps: Map[Dependency, Vector[Dependency]]): Map[Dependency, Vector[Dependency]] = {
val (toRemove, remaining) = reverseDeps.partition(kv => kv._2.isEmpty && !rootDependencies0(kv._1))
if (toRemove.isEmpty) reverseDeps
else helper(remaining.mapValues(_.filter(x => remaining.contains(x) || rootDependencies0(x))).toList.toMap)
}
val filteredReverseDependencies = helper(reverseDependencies)
rootDependencies0 ++ filteredReverseDependencies.keys
}
/**
* The final next dependency set, stripped of no more required ones.
*/
def newDependencies: Set[Dependency] = {
val remainingDependencies0 = remainingDependencies
nextDependenciesAndConflicts._2
.filter(dep => remainingDependencies0(eraseVersion(dep)))
.toSet
}
private def nextNoMissingUnsafe: Resolution = {
val (newConflicts, _) = nextDependenciesAndConflicts
copy(dependencies = newDependencies ++ newConflicts, conflicts = newConflicts.toSet)
}
/**
* If no module info is missing, the next state of the resolution, which can be immediately calculated.
* Else, the current resolution itself.
*/
def nextIfNoMissing: Resolution = {
val missing = missingFromCache
if (missing.isEmpty) nextNoMissingUnsafe
else this
}
/**
* Do a new iteration, fetching the missing modules along the way.
*/
def next(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = {
val missing = missingFromCache
if (missing.isEmpty) Task.now(nextNoMissingUnsafe)
else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing)
}
/**
* Required modules for the dependency management of `project`.
*/
def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = {
val approxProperties =
project.parent
.flatMap(projectsCache.get)
.map(_._2.properties)
.fold(project.properties)(mergeProperties(project.properties, _))
val profileDependencies =
profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation)
.flatMap(_.dependencies)
val modules =
(project.dependencies ++ profileDependencies)
.collect{ case dep if dep.scope == Scope.Import => dep.moduleVersion } ++
project.parent
modules.toSet
}
/**
* Missing modules in cache, to get the full list of dependencies of `project`, taking
* dependency management / inheritance into account.
*
* Note that adding the missing modules to the cache may unveil other missing modules, so
* these modules should be added to the cache, and `dependencyManagementMissing` checked again
* for new missing modules.
*/
def dependencyManagementMissing(project: Project): Set[ModuleVersion] = {
@tailrec
def helper(toCheck: Set[ModuleVersion],
done: Set[ModuleVersion],
missing: Set[ModuleVersion]): Set[ModuleVersion] = {
if (toCheck.isEmpty) missing
else if (toCheck.exists(done)) helper(toCheck -- done, done, missing)
else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing)
else if (toCheck.exists(projectsCache.contains)) {
val (checking, remaining) = toCheck.partition(projectsCache.contains)
val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectsCache(mod)._2))
helper(remaining ++ directRequirements, done ++ checking, missing)
} else if (toCheck.exists(errors.contains)) {
val (errored, remaining) = toCheck.partition(errors.contains)
helper(remaining, done ++ errored, missing)
} else
helper(Set.empty, done, missing ++ toCheck)
}
helper(dependencyManagementRequirements(project), Set(project.moduleVersion), Set.empty)
}
/**
* Add dependency management / inheritance related items to `project`, from what's available in cache.
* It is recommended to have fetched what `dependencyManagementMissing` returned prior to calling
* `withDependencyManagement`.
*/
def withDependencyManagement(project: Project): Project = {
val approxProperties =
project.parent
.filter(projectsCache.contains)
.map(projectsCache(_)._2.properties)
.fold(project.properties)(mergeProperties(project.properties, _))
val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation)
val dependencies0 = addDependencies(project.dependencies +: profiles0.map(_.dependencies))
val properties0 = (project.properties /: profiles0)((acc, p) => mergeProperties(acc, p.properties))
val deps =
dependencies0
.collect{ case dep if dep.scope == Scope.Import && projectsCache.contains(dep.moduleVersion) => dep.moduleVersion } ++
project.parent.filter(projectsCache.contains)
val projs = deps.map(projectsCache(_)._2)
val depMgmt =
(project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement)))
.foldLeft(Map.empty[DepMgmtKey, Dependency])(dependencyManagementAddSeq)
val depsSet = deps.toSet
project.copy(
dependencies = dependencies0
.filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++
project.parent
.filter(projectsCache.contains)
.toSeq
.flatMap(projectsCache(_)._2.dependencies),
dependencyManagement = depMgmt.values.toSeq,
properties = project.parent
.filter(projectsCache.contains)
.map(projectsCache(_)._2.properties)
.fold(properties0)(mergeProperties(properties0, _))
)
}
/**
* Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache.
*/
def fetch(modules: Seq[ModuleVersion],
fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = {
val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _))
val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true)
gatheredLookups.flatMap{ lookupResults =>
val errors0 = errors ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors}
val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj}
/*
* newProjects are project definitions, fresh from the repositories. We need to add
* dependency management / inheritance-related bits to them.
*/
newProjects.foldLeft(Task.now(copy(errors = errors0))) { case (accTask, (modVer, (repo, proj))) =>
for {
current <- accTask
updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule)
proj0 = updated.withDependencyManagement(proj)
} yield updated.copy(projectsCache = updated.projectsCache + (modVer -> (repo, proj0)))
}
}
}
}

View File

@ -68,9 +68,9 @@ package object coursier {
type Repository = core.Repository
def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, List[String], (Repository, Project)] =
modVersion => core.Resolver.find(repositories, modVersion._1, modVersion._2)
modVersion => core.Resolution.find(repositories, modVersion._1, modVersion._2)
type Resolution = core.Resolver.Resolution
type Resolution = core.Resolution
object Resolution {
val empty = apply()
def apply(rootDependencies: Set[Dependency] = Set.empty,
@ -80,7 +80,7 @@ package object coursier {
errors: Map[ModuleVersion, Seq[String]] = Map.empty,
filter: Option[Dependency => Boolean] = None,
profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution =
core.Resolver.Resolution(rootDependencies, dependencies, conflicts, projectsCache, errors, filter, profileActivation)
core.Resolution(rootDependencies, dependencies, conflicts, projectsCache, errors, filter, profileActivation)
}
def resolve(dependencies: Set[Dependency],
@ -88,6 +88,6 @@ package object coursier {
maxIterations: Option[Int] = Some(200),
filter: Option[Dependency => Boolean] = None,
profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = {
core.Resolver.resolve(dependencies, fetch, maxIterations, filter, profileActivation)
core.Resolution.resolve(dependencies, fetch, maxIterations, filter, profileActivation)
}
}

View File

@ -1,13 +1,12 @@
package coursier
package test
import coursier.core.Resolver
import utest._
import scala.async.Async.{async, await}
import coursier.test.compatibility._
object ResolverTests extends TestSuite {
object ResolutionTests extends TestSuite {
implicit class ProjectOps(val p: Project) extends AnyVal {
def kv: (ModuleVersion, (Repository, Project)) = p.moduleVersion -> (testRepository, p)
@ -496,7 +495,7 @@ object ResolverTests extends TestSuite {
'parts{
'propertySubstitution{
val res =
Resolver.withProperties(
core.Resolution.withProperties(
Seq(Dependency(Module("a-company", "a-name"), "${a.property}")),
Map("a.property" -> "a-version"))
val expected = Seq(Dependency(Module("a-company", "a-name"), "a-version"))