From e6ec32f33a3a48d9c97e6198afd9a7f2465ab4b0 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:46 +0100 Subject: [PATCH 01/12] Back to a simpler artifact model in resolver --- .../main/scala/coursier/cli/Coursier.scala | 6 +---- .../src/main/scala/coursier/core/Remote.scala | 17 +++--------- .../scala/coursier/core/Definitions.scala | 27 +++---------------- .../main/scala/coursier/core/Resolver.scala | 15 ++++------- core/src/main/scala/coursier/core/Xml.scala | 2 +- core/src/main/scala/coursier/package.scala | 18 ++++++------- .../scala/coursier/test/CentralTests.scala | 8 ++---- .../scala/coursier/test/PomParsingTests.scala | 2 +- web/src/main/scala/coursier/web/Backend.scala | 8 ++---- 9 files changed, 28 insertions(+), 75 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 2d2c292f7..a8bc40a0e 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -98,17 +98,13 @@ case class Coursier(scope: List[String], } def repr(dep: Dependency) = { - val (type0, classifier) = dep.artifacts match { - case maven: Artifacts.Maven => (maven.`type`, maven.classifier) - } - // dep.version can be an interval, whereas the one from project can't val version = res.projectsCache.get(dep.moduleVersion).map(_._2.version).getOrElse(dep.version) val extra = if (version == dep.version) "" else s" ($version for ${dep.version})" - s"${dep.module.organization}:${dep.module.name}:$type0:${Some(classifier).filter(_.nonEmpty).map(_+":").mkString}$version$extra" + s"${dep.module.organization}:${dep.module.name}:${dep.artifact.`type`}:${Some(dep.artifact.classifier).filter(_.nonEmpty).map(_+":").mkString}$version$extra" } val trDeps = res.dependencies.toList.sortBy(repr) diff --git a/core-jvm/src/main/scala/coursier/core/Remote.scala b/core-jvm/src/main/scala/coursier/core/Remote.scala index f9008f2b4..414a8a00f 100644 --- a/core-jvm/src/main/scala/coursier/core/Remote.scala +++ b/core-jvm/src/main/scala/coursier/core/Remote.scala @@ -21,7 +21,7 @@ case class ArtifactDownloader(root: String, cache: File, logger: Option[Artifact def artifact(module: Module, version: String, - artifact: Artifacts.Artifact, + artifact: Dependency.MavenArtifact, cachePolicy: CachePolicy): EitherT[Task, String, File] = { val relPath = @@ -95,19 +95,8 @@ case class ArtifactDownloader(root: String, cache: File, logger: Option[Artifact project: Project, cachePolicy: CachePolicy = CachePolicy.Default): Task[Seq[String \/ File]] = { - val artifacts0 = - dependency.artifacts match { - case s: Artifacts.Sufficient => s.artifacts - case p: Artifacts.WithProject => p.artifacts(project) - } - - val tasks = - artifacts0 .map { artifact0 => - // Important: using version from project, as the one from dependency can be an interval - artifact(dependency.module, project.version, artifact0, cachePolicy = cachePolicy).run - } - - Task.gatherUnordered(tasks) + // Important: using version from project, as the one from dependency can be an interval + artifact(dependency.module, project.version, dependency.artifact, cachePolicy = cachePolicy).run.map(Seq(_)) } } diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index ee48bd04e..b0836f7f7 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -32,34 +32,15 @@ sealed abstract class Scope(val name: String) case class Dependency(module: Module, version: String, scope: Scope, - artifacts: Artifacts, + artifact: Dependency.MavenArtifact, exclusions: Set[(String, String)], optional: Boolean) { def moduleVersion = (module, version) } -sealed trait Artifacts - -object Artifacts { - /** - * May become a bit more complicated with Ivy support, - * but should still point at one single artifact. - */ - case class Artifact(`type`: String, - classifier: String) - - sealed trait WithProject extends Artifacts { - def artifacts(project: Project): Seq[Artifact] - } - - sealed trait Sufficient extends Artifacts { - def artifacts: Seq[Artifact] - } - - case class Maven(`type`: String, - classifier: String) extends Sufficient { - def artifacts: Seq[Artifact] = Seq(Artifact(`type`, classifier)) - } +object Dependency { + case class MavenArtifact(`type`: String, + classifier: String) } case class Project(module: Module, diff --git a/core/src/main/scala/coursier/core/Resolver.scala b/core/src/main/scala/coursier/core/Resolver.scala index 0c2bd39da..936bc21d2 100644 --- a/core/src/main/scala/coursier/core/Resolver.scala +++ b/core/src/main/scala/coursier/core/Resolver.scala @@ -70,9 +70,7 @@ object Resolver { type DepMgmtKey = (String, String, String) def dependencyManagementKey(dep: Dependency): DepMgmtKey = - dep.artifacts match { - case Artifacts.Maven(type0, _) => (dep.module.organization, dep.module.name, type0) - } + (dep.module.organization, dep.module.name, dep.artifact.`type`) def dependencyManagementAdd(m: Map[DepMgmtKey, Dependency], dep: Dependency): Map[DepMgmtKey, Dependency] = { val key = dependencyManagementKey(dep) if (m.contains(key)) m else m + (key -> dep) @@ -121,13 +119,10 @@ object Resolver { name = substituteProps(dep.module.name) ), version = substituteProps(dep.version), - artifacts = dep.artifacts match { - case maven: Artifacts.Maven => - maven.copy( - `type` = substituteProps(maven.`type`), - classifier = substituteProps(maven.classifier) - ) - }, + artifact = dep.artifact.copy( + `type` = substituteProps(dep.artifact.`type`), + classifier = substituteProps(dep.artifact.classifier) + ), scope = Parse.scope(substituteProps(dep.scope.name)), exclusions = dep.exclusions .map{case (org, name) => (substituteProps(org), substituteProps(name))} diff --git a/core/src/main/scala/coursier/core/Xml.scala b/core/src/main/scala/coursier/core/Xml.scala index 621b3272f..2c92a03bc 100644 --- a/core/src/main/scala/coursier/core/Xml.scala +++ b/core/src/main/scala/coursier/core/Xml.scala @@ -85,7 +85,7 @@ object Xml { mod, version0, scopeOpt getOrElse defaultScope, - Artifacts.Maven(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), + Dependency.MavenArtifact(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), exclusions.map(mod => (mod.organization, mod.name)).toSet, optional ) diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 01a2083b5..f4bbcde21 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -1,6 +1,9 @@ import scalaz.EitherT import scalaz.concurrent.Task +/** + * Pulls definitions from coursier.core, with default arguments. + */ package object coursier { type Dependency = core.Dependency @@ -8,19 +11,16 @@ package object coursier { def apply(module: Module, version: String, scope: Scope = Scope.Other(""), // Substituted by Resolver with its own default scope (compile) - artifacts: Artifacts = Artifacts.Maven(), + artifact: MavenArtifact = MavenArtifact(), exclusions: Set[(String, String)] = Set.empty, optional: Boolean = false): Dependency = - core.Dependency(module, version, scope, artifacts, exclusions, optional) - } + core.Dependency(module, version, scope, artifact, exclusions, optional) - type Artifacts = core.Artifacts - object Artifacts { - type Maven = core.Artifacts.Maven - object Maven { + type MavenArtifact = core.Dependency.MavenArtifact + object MavenArtifact { def apply(`type`: String = "jar", - classifier: String = ""): Maven = - core.Artifacts.Maven(`type`, classifier) + classifier: String = ""): MavenArtifact = + core.Dependency.MavenArtifact(`type`, classifier) } } diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 1d58286d6..9e7683f82 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -12,12 +12,8 @@ object CentralTests extends TestSuite { repository.mavenCentral ) - def repr(dep: Dependency) = { - val (type0, classifier) = dep.artifacts match { - case maven: Artifacts.Maven => (maven.`type`, maven.classifier) - } - s"${dep.module.organization}:${dep.module.name}:$type0:${Some(classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" - } + def repr(dep: Dependency) = + s"${dep.module.organization}:${dep.module.name}:${dep.artifact.`type`}:${Some(dep.artifact.classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" def resolutionCheck(module: Module, version: String) = async { diff --git a/core/src/test/scala/coursier/test/PomParsingTests.scala b/core/src/test/scala/coursier/test/PomParsingTests.scala index 83d339acf..8b3d9fbf9 100644 --- a/core/src/test/scala/coursier/test/PomParsingTests.scala +++ b/core/src/test/scala/coursier/test/PomParsingTests.scala @@ -21,7 +21,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Dependency(Module("comp", "lib"), "2.1", artifacts = Artifacts.Maven(classifier = "extra"))) + val expected = \/-(Dependency(Module("comp", "lib"), "2.1", artifact = Dependency.MavenArtifact(classifier = "extra"))) val result = Xml.dependency(xmlParse(depNode).right.get) diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 1b5c73f57..c37f491e0 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -218,10 +218,6 @@ object App { ) def depItem(dep: Dependency, finalVersionOpt: Option[String]) = { - val (type0, classifier) = dep.artifacts match { - case maven: Artifacts.Maven => (maven.`type`, maven.classifier) - } - <.tr( ^.`class` := (if (res.errors.contains(dep.moduleVersion)) "danger" else ""), <.td(dep.module.organization), @@ -229,8 +225,8 @@ object App { <.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")), <.td(Seq[Seq[TagMod]]( if (dep.scope == Scope.Compile) Seq() else Seq(infoLabel(dep.scope.name)), - if (type0.isEmpty || type0 == "jar") Seq() else Seq(infoLabel(type0)), - if (classifier.isEmpty) Seq() else Seq(infoLabel(classifier)), + if (dep.artifact.`type`.isEmpty || dep.artifact.`type` == "jar") Seq() else Seq(infoLabel(dep.artifact.`type`)), + if (dep.artifact.classifier.isEmpty) Seq() else Seq(infoLabel(dep.artifact.classifier)), Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq, if (dep.optional) Seq(infoLabel("optional")) else Seq(), res.errors.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq From 54338f7b049d9f18907468218e1713658e6112ec Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:47 +0100 Subject: [PATCH 02/12] Refactor exclusions, add partial orders / minifications --- .../main/scala/coursier/core/Exclusions.scala | 53 +++++++ .../src/main/scala/coursier/core/Orders.scala | 142 ++++++++++++++++++ .../main/scala/coursier/core/Resolver.scala | 44 +----- .../scala/coursier/test/ExclusionsTests.scala | 4 +- 4 files changed, 201 insertions(+), 42 deletions(-) create mode 100644 core/src/main/scala/coursier/core/Exclusions.scala create mode 100644 core/src/main/scala/coursier/core/Orders.scala diff --git a/core/src/main/scala/coursier/core/Exclusions.scala b/core/src/main/scala/coursier/core/Exclusions.scala new file mode 100644 index 000000000..742961223 --- /dev/null +++ b/core/src/main/scala/coursier/core/Exclusions.scala @@ -0,0 +1,53 @@ +package coursier.core + +object Exclusions { + + def partition(exclusions: Set[(String, String)]): (Boolean, Set[String], Set[String], Set[(String, String)]) = { + + val (wildCards, remaining) = exclusions + .partition{case (org, name) => org == "*" || name == "*" } + + val all = wildCards + .contains(("*", "*")) + + val excludeByOrg = wildCards + .collect{case (org, "*") if org != "*" => org } + val excludeByName = wildCards + .collect{case ("*", name) if name != "*" => name } + + (all, excludeByOrg, excludeByName, remaining) + } + + def apply(exclusions: Set[(String, String)]): (String, String) => Boolean = { + + val (all, excludeByOrg, excludeByName, remaining) = partition(exclusions) + + if (all) (_, _) => false + else + (org, name) => { + !excludeByName(name) && + !excludeByOrg(org) && + !remaining((org, name)) + } + } + + def minimize(exclusions: Set[(String, String)]): Set[(String, String)] = { + + val (all, excludeByOrg, excludeByName, remaining) = partition(exclusions) + + if (all) Set(("*", "*")) + else { + val filteredRemaining = remaining + .filter{case (org, name) => + !excludeByOrg(org) && + !excludeByName(name) + } + + excludeByOrg.map((_, "*")) ++ + excludeByName.map(("*", _)) ++ + filteredRemaining + } + } + + +} diff --git a/core/src/main/scala/coursier/core/Orders.scala b/core/src/main/scala/coursier/core/Orders.scala new file mode 100644 index 000000000..1653e99d7 --- /dev/null +++ b/core/src/main/scala/coursier/core/Orders.scala @@ -0,0 +1,142 @@ +package coursier.core + +object Orders { + + /** Minimal ad-hoc partial order */ + trait PartialOrder[A] { + /** + * x < y: Some(neg. integer) + * x == y: Some(0) + * x > y: Some(pos. integer) + * x, y not related: None + */ + def cmp(x: A, y: A): Option[Int] + } + + /** + * Only relations: + * Compile < Runtime < Test + */ + implicit val mavenScopePartialOrder: PartialOrder[Scope] = + new PartialOrder[Scope] { + val higher = Map[Scope, Set[Scope]]( + Scope.Compile -> Set(Scope.Runtime, Scope.Test), + Scope.Runtime -> Set(Scope.Test) + ) + + def cmp(x: Scope, y: Scope) = + if (x == y) Some(0) + else if (higher.get(x).exists(_(y))) Some(-1) + else if (higher.get(y).exists(_(x))) Some(1) + else None + } + + /** Non-optional < optional */ + implicit val optionalPartialOrder: PartialOrder[Boolean] = + new PartialOrder[Boolean] { + def cmp(x: Boolean, y: Boolean) = + Some( + if (x == y) 0 + else if (x) 1 + else -1 + ) + } + + /** + * Exclusions partial order. + * + * x <= y iff all that x excludes is also excluded by y. + * x and y not related iff x excludes some elements not excluded by y AND + * y excludes some elements not excluded by x. + * + * In particular, no exclusions <= anything <= Set(("*", "*")) + */ + implicit val exclusionsPartialOrder: PartialOrder[Set[(String, String)]] = + new PartialOrder[Set[(String, String)]] { + def boolCmp(a: Boolean, b: Boolean) = (a, b) match { + case (true, true) => Some(0) + case (true, false) => Some(1) + case (false, true) => Some(-1) + case (false, false) => None + } + + def cmp(x: Set[(String, String)], y: Set[(String, String)]) = { + val (xAll, xExcludeByOrg1, xExcludeByName1, xRemaining0) = Exclusions.partition(x) + val (yAll, yExcludeByOrg1, yExcludeByName1, yRemaining0) = Exclusions.partition(y) + + boolCmp(xAll, yAll).orElse { + def filtered(e: Set[(String, String)]) = + e.filter{case (org, name) => + !xExcludeByOrg1(org) && !yExcludeByOrg1(org) && + !xExcludeByName1(name) && !yExcludeByName1(name) + } + + def removeIntersection[T](a: Set[T], b: Set[T]) = + (a -- b, b -- a) + + def allEmpty(set: Set[_]*) = set.forall(_.isEmpty) + + val (xRemaining1, yRemaining1) = + (filtered(xRemaining0), filtered(yRemaining0)) + + val (xProperRemaining, yProperRemaining) = + removeIntersection(xRemaining1, yRemaining1) + + val (onlyXExcludeByOrg, onlyYExcludeByOrg) = + removeIntersection(xExcludeByOrg1, yExcludeByOrg1) + + val (onlyXExcludeByName, onlyYExcludeByName) = + removeIntersection(xExcludeByName1, yExcludeByName1) + + val (noXProper, noYProper) = ( + allEmpty(xProperRemaining, onlyXExcludeByOrg, onlyXExcludeByName), + allEmpty(yProperRemaining, onlyYExcludeByOrg, onlyYExcludeByName) + ) + + boolCmp(noYProper, noXProper) // order matters + } + } + } + + /** + * Assume all dependencies have same `module`, `version`, and `artifact`; see `minDependencies` + * if they don't. + */ + def minDependenciesUnsafe(dependencies: Set[Dependency]): Set[Dependency] = { + val groupedDependencies = dependencies + .groupBy(dep => (dep.optional, dep.scope)) + .toList + + val remove = + for { + List(((xOpt, xScope), xDeps), ((yOpt, yScope), yDeps)) <- groupedDependencies.combinations(2) + optCmp <- optionalPartialOrder.cmp(xOpt, yOpt).iterator + scopeCmp <- mavenScopePartialOrder.cmp(xScope, yScope).iterator + if optCmp*scopeCmp >= 0 + xDep <- xDeps.iterator + yDep <- yDeps.iterator + exclCmp <- exclusionsPartialOrder.cmp(xDep.exclusions, yDep.exclusions).iterator + if optCmp*exclCmp >= 0 + if scopeCmp*exclCmp >= 0 + xIsMin = optCmp < 0 || scopeCmp < 0 || exclCmp < 0 + yIsMin = optCmp > 0 || scopeCmp > 0 || exclCmp > 0 + if xIsMin || yIsMin // should be always true, unless xDep == yDep, which shouldn't happen + } yield if (xIsMin) yDep else xDep + + dependencies -- remove + } + + /** + * Minified representation of `dependencies`. + * + * The returned set brings exactly the same things as `dependencies`, with no redundancy. + */ + def minDependencies(dependencies: Set[Dependency]): Set[Dependency] = { + dependencies + .groupBy(_.copy(scope = Scope.Other(""), exclusions = Set.empty, optional = false)) + .mapValues(minDependenciesUnsafe) + .valuesIterator + .fold(Set.empty)(_ ++ _) + } + +} diff --git a/core/src/main/scala/coursier/core/Resolver.scala b/core/src/main/scala/coursier/core/Resolver.scala index 936bc21d2..035c047aa 100644 --- a/core/src/main/scala/coursier/core/Resolver.scala +++ b/core/src/main/scala/coursier/core/Resolver.scala @@ -223,34 +223,6 @@ object Resolver { } } - /** - * Addition of exclusions. A module is excluded by the result if it is excluded - * by `first`, by `second`, or by both. - */ - def exclusionsAdd(first: Set[(String, String)], - second: Set[(String, String)]): Set[(String, String)] = { - - val (firstAll, firstNonAll) = first.partition{case ("*", "*") => true; case _ => false } - val (secondAll, secondNonAll) = second.partition{case ("*", "*") => true; case _ => false } - - if (firstAll.nonEmpty || secondAll.nonEmpty) Set(("*", "*")) - else { - val firstOrgWildcards = firstNonAll.collect{ case ("*", name) => name } - val firstNameWildcards = firstNonAll.collect{ case (org, "*") => org } - val secondOrgWildcards = secondNonAll.collect{ case ("*", name) => name } - val secondNameWildcards = secondNonAll.collect{ case (org, "*") => org } - - val orgWildcards = firstOrgWildcards ++ secondOrgWildcards - val nameWildcards = firstNameWildcards ++ secondNameWildcards - - val firstRemaining = firstNonAll.filter{ case (org, name) => org != "*" && name != "*" } - val secondRemaining = secondNonAll.filter{ case (org, name) => org != "*" && name != "*" } - - val remaining = (firstRemaining ++ secondRemaining).filterNot{case (org, name) => orgWildcards(name) || nameWildcards(org) } - - orgWildcards.map(name => ("*", name)) ++ nameWildcards.map(org => (org, "*")) ++ remaining - } - } def withDefaultScope(dep: Dependency): Dependency = if (dep.scope.name.isEmpty) dep.copy(scope = Scope.Compile) @@ -262,22 +234,12 @@ object Resolver { def withExclusions(dependencies: Seq[Dependency], exclusions: Set[(String, String)]): Seq[Dependency] = { - val (all, notAll) = exclusions.partition{case ("*", "*") => true; case _ => false} - - val orgWildcards = notAll.collect{case ("*", name) => name } - val nameWildcards = notAll.collect{case (org, "*") => org } - - val remaining = notAll.filterNot{case (org, name) => org == "*" || name == "*" } + val filter = Exclusions(exclusions) dependencies - .filter(dep => - all.isEmpty && - !orgWildcards(dep.module.name) && - !nameWildcards(dep.module.organization) && - !remaining((dep.module.organization, dep.module.name)) - ) + .filter(dep => filter(dep.module.organization, dep.module.name)) .map(dep => - dep.copy(exclusions = exclusionsAdd(dep.exclusions, exclusions)) + dep.copy(exclusions = Exclusions.minimize(dep.exclusions ++ exclusions)) ) } diff --git a/core/src/test/scala/coursier/test/ExclusionsTests.scala b/core/src/test/scala/coursier/test/ExclusionsTests.scala index 3eb775d95..051421384 100644 --- a/core/src/test/scala/coursier/test/ExclusionsTests.scala +++ b/core/src/test/scala/coursier/test/ExclusionsTests.scala @@ -2,10 +2,12 @@ package coursier package test import utest._ -import core.Resolver.exclusionsAdd object ExclusionsTests extends TestSuite { + def exclusionsAdd(e1: Set[(String, String)], e2: Set[(String, String)]) = + core.Exclusions.minimize(e1 ++ e2) + val tests = TestSuite { val e1 = Set(("org1", "name1")) val e2 = Set(("org2", "name2")) From 0de5330351b09bff6628e43ed23d1e6150105f1b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:49 +0100 Subject: [PATCH 03/12] Small refactoring --- .../core/{Resolver.scala => Resolution.scala} | 573 +++++++++--------- core/src/main/scala/coursier/package.scala | 8 +- ...olverTests.scala => ResolutionTests.scala} | 5 +- 3 files changed, 293 insertions(+), 293 deletions(-) rename core/src/main/scala/coursier/core/{Resolver.scala => Resolution.scala} (53%) rename core/src/test/scala/coursier/test/{ResolverTests.scala => ResolutionTests.scala} (99%) diff --git a/core/src/main/scala/coursier/core/Resolver.scala b/core/src/main/scala/coursier/core/Resolution.scala similarity index 53% rename from core/src/main/scala/coursier/core/Resolver.scala rename to core/src/main/scala/coursier/core/Resolution.scala index 035c047aa..4ccabce8f 100644 --- a/core/src/main/scala/coursier/core/Resolver.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -7,7 +7,7 @@ import scala.collection.mutable import scalaz.concurrent.Task import scalaz.{EitherT, \/-, \/, -\/} -object Resolver { +object Resolution { type ModuleVersion = (Module, String) @@ -283,291 +283,6 @@ object Resolver { } } - /** - * State of a dependency resolution. - * - * Done if method `isDone` returns `true`. - * - * @param dependencies: current set of dependencies - * @param conflicts: conflicting dependencies - * @param projectsCache: cache of known projects - * @param errors: keeps track of the modules whose project definition could not be found - */ - case class Resolution(rootDependencies: Set[Dependency], - dependencies: Set[Dependency], - conflicts: Set[Dependency], - projectsCache: Map[ModuleVersion, (Repository, Project)], - errors: Map[ModuleVersion, Seq[String]], - filter: Option[Dependency => Boolean], - profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) { - - private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]() - private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized { - finalDependenciesCache.getOrElseUpdate(dep, - projectsCache.get(dep.moduleVersion) match { - case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter) - case None => Nil - } - ) - } - - /** - * Transitive dependencies of the current dependencies, according to what there currently is in cache. - * No attempt is made to solve version conflicts here. - */ - def transitiveDependencies: Seq[Dependency] = - for { - dep <- (dependencies -- conflicts).toList - trDep <- finalDependencies0(dep) - } yield trDep - - /** - * The "next" dependency set, made of the current dependencies and their transitive dependencies, - * trying to solve version conflicts. Transitive dependencies are calculated with the current cache. - * - * May contain dependencies added in previous iterations, but no more required. These are filtered below, see - * `newDependencies`. - * - * Returns a tuple made of the conflicting dependencies, and all the dependencies. - */ - def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = { - merge(dependencies ++ transitiveDependencies) - } - - /** - * The modules we miss some info about. - */ - def missingFromCache: Set[ModuleVersion] = { - val modules = dependencies.map(_.moduleVersion) - val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion) - - (modules ++ nextModules) - .filterNot(mod => projectsCache.contains(mod) || errors.contains(mod)) - } - - - /** - * Whether the resolution is done. - */ - def isDone: Boolean = { - def isFixPoint = { - val (nextConflicts, _) = nextDependenciesAndConflicts - dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet - } - - missingFromCache.isEmpty && isFixPoint - } - - private def eraseVersion(dep: Dependency) = dep.copy(version = "") - - /** - * Returns a map giving the dependencies that brought each of the dependency of the "next" dependency set. - * - * The versions of all the dependencies returned are erased (emptied). - */ - def reverseDependencies: Map[Dependency, Vector[Dependency]] = { - val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts - - val trDepsSeq = - for { - dep <- updatedDeps - trDep <- finalDependencies0(dep) - } yield eraseVersion(trDep) -> eraseVersion(dep) - - val knownDeps = (updatedDeps ++ updatedConflicts).map(eraseVersion).toSet - - trDepsSeq - .groupBy(_._1) - .mapValues(_.map(_._2).toVector) - .filterKeys(knownDeps) - .toList.toMap // Eagerly evaluate filterKeys/mapValues - } - - /** - * Returns dependencies from the "next" dependency set, filtering out - * those that are no more required. - * - * The versions of all the dependencies returned are erased (emptied). - */ - def remainingDependencies: Set[Dependency] = { - val rootDependencies0 = rootDependencies.map(eraseVersion) - - @tailrec - def helper(reverseDeps: Map[Dependency, Vector[Dependency]]): Map[Dependency, Vector[Dependency]] = { - val (toRemove, remaining) = reverseDeps.partition(kv => kv._2.isEmpty && !rootDependencies0(kv._1)) - - if (toRemove.isEmpty) reverseDeps - else helper(remaining.mapValues(_.filter(x => remaining.contains(x) || rootDependencies0(x))).toList.toMap) - } - - val filteredReverseDependencies = helper(reverseDependencies) - - rootDependencies0 ++ filteredReverseDependencies.keys - } - - /** - * The final next dependency set, stripped of no more required ones. - */ - def newDependencies: Set[Dependency] = { - val remainingDependencies0 = remainingDependencies - nextDependenciesAndConflicts._2 - .filter(dep => remainingDependencies0(eraseVersion(dep))) - .toSet - } - - private def nextNoMissingUnsafe: Resolution = { - val (newConflicts, _) = nextDependenciesAndConflicts - copy(dependencies = newDependencies ++ newConflicts, conflicts = newConflicts.toSet) - } - - /** - * If no module info is missing, the next state of the resolution, which can be immediately calculated. - * Else, the current resolution itself. - */ - def nextIfNoMissing: Resolution = { - val missing = missingFromCache - if (missing.isEmpty) nextNoMissingUnsafe - else this - } - - /** - * Do a new iteration, fetching the missing modules along the way. - */ - def next(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { - val missing = missingFromCache - if (missing.isEmpty) Task.now(nextNoMissingUnsafe) - else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing) - } - - /** - * Required modules for the dependency management of `project`. - */ - def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = { - val approxProperties = - project.parent - .flatMap(projectsCache.get) - .map(_._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) - - val profileDependencies = - profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) - .flatMap(_.dependencies) - - val modules = - (project.dependencies ++ profileDependencies) - .collect{ case dep if dep.scope == Scope.Import => dep.moduleVersion } ++ - project.parent - - modules.toSet - } - - /** - * Missing modules in cache, to get the full list of dependencies of `project`, taking - * dependency management / inheritance into account. - * - * Note that adding the missing modules to the cache may unveil other missing modules, so - * these modules should be added to the cache, and `dependencyManagementMissing` checked again - * for new missing modules. - */ - def dependencyManagementMissing(project: Project): Set[ModuleVersion] = { - - @tailrec - def helper(toCheck: Set[ModuleVersion], - done: Set[ModuleVersion], - missing: Set[ModuleVersion]): Set[ModuleVersion] = { - - if (toCheck.isEmpty) missing - else if (toCheck.exists(done)) helper(toCheck -- done, done, missing) - else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing) - else if (toCheck.exists(projectsCache.contains)) { - val (checking, remaining) = toCheck.partition(projectsCache.contains) - val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectsCache(mod)._2)) - - helper(remaining ++ directRequirements, done ++ checking, missing) - } else if (toCheck.exists(errors.contains)) { - val (errored, remaining) = toCheck.partition(errors.contains) - helper(remaining, done ++ errored, missing) - } else - helper(Set.empty, done, missing ++ toCheck) - } - - helper(dependencyManagementRequirements(project), Set(project.moduleVersion), Set.empty) - } - - /** - * Add dependency management / inheritance related items to `project`, from what's available in cache. - * It is recommended to have fetched what `dependencyManagementMissing` returned prior to calling - * `withDependencyManagement`. - */ - def withDependencyManagement(project: Project): Project = { - - val approxProperties = - project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) - - val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) - - val dependencies0 = addDependencies(project.dependencies +: profiles0.map(_.dependencies)) - val properties0 = (project.properties /: profiles0)((acc, p) => mergeProperties(acc, p.properties)) - - val deps = - dependencies0 - .collect{ case dep if dep.scope == Scope.Import && projectsCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ - project.parent.filter(projectsCache.contains) - val projs = deps.map(projectsCache(_)._2) - - val depMgmt = - (project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement))) - .foldLeft(Map.empty[DepMgmtKey, Dependency])(dependencyManagementAddSeq) - - val depsSet = deps.toSet - - project.copy( - dependencies = dependencies0 - .filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++ - project.parent - .filter(projectsCache.contains) - .toSeq - .flatMap(projectsCache(_)._2.dependencies), - dependencyManagement = depMgmt.values.toSeq, - properties = project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) - .fold(properties0)(mergeProperties(properties0, _)) - ) - } - - /** - * Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache. - */ - def fetch(modules: Seq[ModuleVersion], - fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { - - val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) - val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) - gatheredLookups.flatMap{ lookupResults => - val errors0 = errors ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} - val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj} - - /* - * newProjects are project definitions, fresh from the repositories. We need to add - * dependency management / inheritance-related bits to them. - */ - - newProjects.foldLeft(Task.now(copy(errors = errors0))) { case (accTask, (modVer, (repo, proj))) => - for { - current <- accTask - updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule) - proj0 = updated.withDependencyManagement(proj) - } yield updated.copy(projectsCache = updated.projectsCache + (modVer -> (repo, proj0))) - } - } - } - - } - /** * Default function checking whether a profile is active, given its id, activation conditions, * and the properties of its project. @@ -630,5 +345,291 @@ object Resolver { helper(startResolution, maxIterations).map(_._1) } +} + + + /** + * State of a dependency resolution. + * + * Done if method `isDone` returns `true`. + * + * @param dependencies: current set of dependencies + * @param conflicts: conflicting dependencies + * @param projectsCache: cache of known projects + * @param errors: keeps track of the modules whose project definition could not be found + */ +case class Resolution(rootDependencies: Set[Dependency], + dependencies: Set[Dependency], + conflicts: Set[Dependency], + projectsCache: Map[Resolution.ModuleVersion, (Repository, Project)], + errors: Map[Resolution.ModuleVersion, Seq[String]], + filter: Option[Dependency => Boolean], + profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) { + import Resolution._ + + private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]() + private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized { + finalDependenciesCache.getOrElseUpdate(dep, + projectsCache.get(dep.moduleVersion) match { + case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter) + case None => Nil + } + ) + } + + /** + * Transitive dependencies of the current dependencies, according to what there currently is in cache. + * No attempt is made to solve version conflicts here. + */ + def transitiveDependencies: Seq[Dependency] = + for { + dep <- (dependencies -- conflicts).toList + trDep <- finalDependencies0(dep) + } yield trDep + + /** + * The "next" dependency set, made of the current dependencies and their transitive dependencies, + * trying to solve version conflicts. Transitive dependencies are calculated with the current cache. + * + * May contain dependencies added in previous iterations, but no more required. These are filtered below, see + * `newDependencies`. + * + * Returns a tuple made of the conflicting dependencies, and all the dependencies. + */ + def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = { + merge(dependencies ++ transitiveDependencies) + } + + /** + * The modules we miss some info about. + */ + def missingFromCache: Set[ModuleVersion] = { + val modules = dependencies.map(_.moduleVersion) + val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion) + + (modules ++ nextModules) + .filterNot(mod => projectsCache.contains(mod) || errors.contains(mod)) + } + + + /** + * Whether the resolution is done. + */ + def isDone: Boolean = { + def isFixPoint = { + val (nextConflicts, _) = nextDependenciesAndConflicts + dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet + } + + missingFromCache.isEmpty && isFixPoint + } + + private def eraseVersion(dep: Dependency) = dep.copy(version = "") + + /** + * Returns a map giving the dependencies that brought each of the dependency of the "next" dependency set. + * + * The versions of all the dependencies returned are erased (emptied). + */ + def reverseDependencies: Map[Dependency, Vector[Dependency]] = { + val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts + + val trDepsSeq = + for { + dep <- updatedDeps + trDep <- finalDependencies0(dep) + } yield eraseVersion(trDep) -> eraseVersion(dep) + + val knownDeps = (updatedDeps ++ updatedConflicts).map(eraseVersion).toSet + + trDepsSeq + .groupBy(_._1) + .mapValues(_.map(_._2).toVector) + .filterKeys(knownDeps) + .toList.toMap // Eagerly evaluate filterKeys/mapValues + } + + /** + * Returns dependencies from the "next" dependency set, filtering out + * those that are no more required. + * + * The versions of all the dependencies returned are erased (emptied). + */ + def remainingDependencies: Set[Dependency] = { + val rootDependencies0 = rootDependencies.map(eraseVersion) + + @tailrec + def helper(reverseDeps: Map[Dependency, Vector[Dependency]]): Map[Dependency, Vector[Dependency]] = { + val (toRemove, remaining) = reverseDeps.partition(kv => kv._2.isEmpty && !rootDependencies0(kv._1)) + + if (toRemove.isEmpty) reverseDeps + else helper(remaining.mapValues(_.filter(x => remaining.contains(x) || rootDependencies0(x))).toList.toMap) + } + + val filteredReverseDependencies = helper(reverseDependencies) + + rootDependencies0 ++ filteredReverseDependencies.keys + } + + /** + * The final next dependency set, stripped of no more required ones. + */ + def newDependencies: Set[Dependency] = { + val remainingDependencies0 = remainingDependencies + nextDependenciesAndConflicts._2 + .filter(dep => remainingDependencies0(eraseVersion(dep))) + .toSet + } + + private def nextNoMissingUnsafe: Resolution = { + val (newConflicts, _) = nextDependenciesAndConflicts + copy(dependencies = newDependencies ++ newConflicts, conflicts = newConflicts.toSet) + } + + /** + * If no module info is missing, the next state of the resolution, which can be immediately calculated. + * Else, the current resolution itself. + */ + def nextIfNoMissing: Resolution = { + val missing = missingFromCache + if (missing.isEmpty) nextNoMissingUnsafe + else this + } + + /** + * Do a new iteration, fetching the missing modules along the way. + */ + def next(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { + val missing = missingFromCache + if (missing.isEmpty) Task.now(nextNoMissingUnsafe) + else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing) + } + + /** + * Required modules for the dependency management of `project`. + */ + def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = { + val approxProperties = + project.parent + .flatMap(projectsCache.get) + .map(_._2.properties) + .fold(project.properties)(mergeProperties(project.properties, _)) + + val profileDependencies = + profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) + .flatMap(_.dependencies) + + val modules = + (project.dependencies ++ profileDependencies) + .collect{ case dep if dep.scope == Scope.Import => dep.moduleVersion } ++ + project.parent + + modules.toSet + } + + /** + * Missing modules in cache, to get the full list of dependencies of `project`, taking + * dependency management / inheritance into account. + * + * Note that adding the missing modules to the cache may unveil other missing modules, so + * these modules should be added to the cache, and `dependencyManagementMissing` checked again + * for new missing modules. + */ + def dependencyManagementMissing(project: Project): Set[ModuleVersion] = { + + @tailrec + def helper(toCheck: Set[ModuleVersion], + done: Set[ModuleVersion], + missing: Set[ModuleVersion]): Set[ModuleVersion] = { + + if (toCheck.isEmpty) missing + else if (toCheck.exists(done)) helper(toCheck -- done, done, missing) + else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing) + else if (toCheck.exists(projectsCache.contains)) { + val (checking, remaining) = toCheck.partition(projectsCache.contains) + val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectsCache(mod)._2)) + + helper(remaining ++ directRequirements, done ++ checking, missing) + } else if (toCheck.exists(errors.contains)) { + val (errored, remaining) = toCheck.partition(errors.contains) + helper(remaining, done ++ errored, missing) + } else + helper(Set.empty, done, missing ++ toCheck) + } + + helper(dependencyManagementRequirements(project), Set(project.moduleVersion), Set.empty) + } + + /** + * Add dependency management / inheritance related items to `project`, from what's available in cache. + * It is recommended to have fetched what `dependencyManagementMissing` returned prior to calling + * `withDependencyManagement`. + */ + def withDependencyManagement(project: Project): Project = { + + val approxProperties = + project.parent + .filter(projectsCache.contains) + .map(projectsCache(_)._2.properties) + .fold(project.properties)(mergeProperties(project.properties, _)) + + val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) + + val dependencies0 = addDependencies(project.dependencies +: profiles0.map(_.dependencies)) + val properties0 = (project.properties /: profiles0)((acc, p) => mergeProperties(acc, p.properties)) + + val deps = + dependencies0 + .collect{ case dep if dep.scope == Scope.Import && projectsCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ + project.parent.filter(projectsCache.contains) + val projs = deps.map(projectsCache(_)._2) + + val depMgmt = + (project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement))) + .foldLeft(Map.empty[DepMgmtKey, Dependency])(dependencyManagementAddSeq) + + val depsSet = deps.toSet + + project.copy( + dependencies = dependencies0 + .filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++ + project.parent + .filter(projectsCache.contains) + .toSeq + .flatMap(projectsCache(_)._2.dependencies), + dependencyManagement = depMgmt.values.toSeq, + properties = project.parent + .filter(projectsCache.contains) + .map(projectsCache(_)._2.properties) + .fold(properties0)(mergeProperties(properties0, _)) + ) + } + + /** + * Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache. + */ + def fetch(modules: Seq[ModuleVersion], + fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { + + val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) + val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) + gatheredLookups.flatMap{ lookupResults => + val errors0 = errors ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} + val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj} + + /* + * newProjects are project definitions, fresh from the repositories. We need to add + * dependency management / inheritance-related bits to them. + */ + + newProjects.foldLeft(Task.now(copy(errors = errors0))) { case (accTask, (modVer, (repo, proj))) => + for { + current <- accTask + updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule) + proj0 = updated.withDependencyManagement(proj) + } yield updated.copy(projectsCache = updated.projectsCache + (modVer -> (repo, proj0))) + } + } + } } diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index f4bbcde21..5a9a0d079 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -68,9 +68,9 @@ package object coursier { type Repository = core.Repository def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, List[String], (Repository, Project)] = - modVersion => core.Resolver.find(repositories, modVersion._1, modVersion._2) + modVersion => core.Resolution.find(repositories, modVersion._1, modVersion._2) - type Resolution = core.Resolver.Resolution + type Resolution = core.Resolution object Resolution { val empty = apply() def apply(rootDependencies: Set[Dependency] = Set.empty, @@ -80,7 +80,7 @@ package object coursier { errors: Map[ModuleVersion, Seq[String]] = Map.empty, filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution = - core.Resolver.Resolution(rootDependencies, dependencies, conflicts, projectsCache, errors, filter, profileActivation) + core.Resolution(rootDependencies, dependencies, conflicts, projectsCache, errors, filter, profileActivation) } def resolve(dependencies: Set[Dependency], @@ -88,6 +88,6 @@ package object coursier { maxIterations: Option[Int] = Some(200), filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { - core.Resolver.resolve(dependencies, fetch, maxIterations, filter, profileActivation) + core.Resolution.resolve(dependencies, fetch, maxIterations, filter, profileActivation) } } diff --git a/core/src/test/scala/coursier/test/ResolverTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala similarity index 99% rename from core/src/test/scala/coursier/test/ResolverTests.scala rename to core/src/test/scala/coursier/test/ResolutionTests.scala index 58f1457c6..9e9b577b3 100644 --- a/core/src/test/scala/coursier/test/ResolverTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -1,13 +1,12 @@ package coursier package test -import coursier.core.Resolver import utest._ import scala.async.Async.{async, await} import coursier.test.compatibility._ -object ResolverTests extends TestSuite { +object ResolutionTests extends TestSuite { implicit class ProjectOps(val p: Project) extends AnyVal { def kv: (ModuleVersion, (Repository, Project)) = p.moduleVersion -> (testRepository, p) @@ -496,7 +495,7 @@ object ResolverTests extends TestSuite { 'parts{ 'propertySubstitution{ val res = - Resolver.withProperties( + core.Resolution.withProperties( Seq(Dependency(Module("a-company", "a-name"), "${a.property}")), Map("a.property" -> "a-version")) val expected = Seq(Dependency(Module("a-company", "a-name"), "a-version")) From 510dd7b2d46809c6caa6405e23c379bd079476fe Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:50 +0100 Subject: [PATCH 04/12] Allow empty initial dependencies in Resolution --- core-jvm/src/main/scala/coursier/core/package.scala | 2 +- core/src/main/scala/coursier/core/Resolution.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core-jvm/src/main/scala/coursier/core/package.scala b/core-jvm/src/main/scala/coursier/core/package.scala index aaf241b9f..3d7851006 100644 --- a/core-jvm/src/main/scala/coursier/core/package.scala +++ b/core-jvm/src/main/scala/coursier/core/package.scala @@ -11,7 +11,7 @@ package object core { profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]): Stream[Resolution] = { val startResolution = Resolution( - dependencies, dependencies, Set.empty, + dependencies, Set.empty, Set.empty, Map.empty, Map.empty, filter, profileActivation diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index 4ccabce8f..33c574d6d 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -330,7 +330,7 @@ object Resolution { val dependencies0 = dependencies.map(withDefaultScope) val startResolution = Resolution( - dependencies0, dependencies0, Set.empty, + dependencies0, Set.empty, Set.empty, Map.empty, Map.empty, filter, profileActivation @@ -397,7 +397,7 @@ case class Resolution(rootDependencies: Set[Dependency], * Returns a tuple made of the conflicting dependencies, and all the dependencies. */ def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = { - merge(dependencies ++ transitiveDependencies) + merge(rootDependencies ++ dependencies ++ transitiveDependencies) } /** From 27c8cc99299f1cf10d417a6c72a39efacb03f824 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:50 +0100 Subject: [PATCH 05/12] Changes in Resolution API --- .../main/scala/coursier/cli/Coursier.scala | 14 ++-- .../main/scala/coursier/core/Resolution.scala | 74 +++++++------------ core/src/main/scala/coursier/package.scala | 10 ++- .../scala/coursier/test/CentralTests.scala | 6 +- .../scala/coursier/test/ResolutionTests.scala | 32 ++++---- .../test/scala/coursier/test/package.scala | 17 +++++ web/src/main/scala/coursier/web/Backend.scala | 27 ++++--- 7 files changed, 93 insertions(+), 87 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index a8bc40a0e..23f7cb2de 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -13,7 +13,7 @@ import scalaz.{-\/, \/-} case class Coursier(scope: List[String], keepOptional: Boolean, fetch: Boolean, - @ExtraName("N") maxIterations: Int) extends App { + @ExtraName("N") maxIterations: Int = 100) extends App { val scopes0 = if (scope.isEmpty) List(Scope.Compile, Scope.Runtime) @@ -85,12 +85,12 @@ case class Coursier(scope: List[String], Dependency(mod, ver, scope = Scope.Runtime) } - val res = resolve( + val startRes = Resolution( deps.toSet, - fetchFrom(repositories), - maxIterations = Some(maxIterations).filter(_ > 0), filter = Some(dep => (keepOptional || !dep.optional) && scopes(dep.scope)) - ).run + ) + + val res = startRes.last(fetchFrom(repositories), maxIterations).run if (!res.isDone) { Console.err.println(s"Maximum number of iteration reached!") @@ -107,7 +107,7 @@ case class Coursier(scope: List[String], s"${dep.module.organization}:${dep.module.name}:${dep.artifact.`type`}:${Some(dep.artifact.classifier).filter(_.nonEmpty).map(_+":").mkString}$version$extra" } - val trDeps = res.dependencies.toList.sortBy(repr) + val trDeps = res.minDependencies.toList.sortBy(repr) println("\n" + trDeps.map(repr).distinct.mkString("\n")) @@ -129,7 +129,7 @@ case class Coursier(scope: List[String], val cachePolicy: CachePolicy = CachePolicy.Default - val m = res.dependencies.groupBy(dep => res.projectsCache.get(dep.moduleVersion).map(_._1)) + val m = res.minDependencies.groupBy(dep => res.projectsCache.get(dep.moduleVersion).map(_._1)) val (notFound, remaining0) = m.partition(_._1.isEmpty) if (notFound.nonEmpty) { val notFound0 = notFound.values.flatten.toList.map(repr).sorted diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index 33c574d6d..f03f5ad67 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -312,52 +312,19 @@ object Resolution { def defaultFilter(dep: Dependency): Boolean = !dep.optional && dep.scope == Scope.Compile - /** - * Get all the transitive dependencies of `dependencies`, solving any dependency version mismatch. - * - * Iteratively fetches the missing info of the current dependencies / add newly discovered dependencies - * to the current ones. The maximum number of such iterations can be bounded with `maxIterations`. - * - * ... - * - */ - def resolve(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], - maxIterations: Option[Int], - filter: Option[Dependency => Boolean], - profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]): Task[Resolution] = { - - val dependencies0 = dependencies.map(withDefaultScope) - - val startResolution = Resolution( - dependencies0, Set.empty, Set.empty, - Map.empty, Map.empty, - filter, - profileActivation - ) - - def helper(resolution: Resolution, remainingIter: Option[Int]): Task[(Resolution, Option[Int])] = { - if (resolution.isDone || remainingIter.exists(_ <= 0)) - Task.now((resolution, remainingIter)) - else - resolution.next(fetch).flatMap(helper(_, remainingIter.map(_ - 1))) - } - - helper(startResolution, maxIterations).map(_._1) - } } - /** - * State of a dependency resolution. - * - * Done if method `isDone` returns `true`. - * - * @param dependencies: current set of dependencies - * @param conflicts: conflicting dependencies - * @param projectsCache: cache of known projects - * @param errors: keeps track of the modules whose project definition could not be found - */ +/** + * State of a dependency resolution. + * + * Done if method `isDone` returns `true`. + * + * @param dependencies: current set of dependencies + * @param conflicts: conflicting dependencies + * @param projectsCache: cache of known projects + * @param errors: keeps track of the modules whose project definition could not be found + */ case class Resolution(rootDependencies: Set[Dependency], dependencies: Set[Dependency], conflicts: Set[Dependency], @@ -382,10 +349,9 @@ case class Resolution(rootDependencies: Set[Dependency], * No attempt is made to solve version conflicts here. */ def transitiveDependencies: Seq[Dependency] = - for { - dep <- (dependencies -- conflicts).toList - trDep <- finalDependencies0(dep) - } yield trDep + (dependencies -- conflicts) + .toList + .flatMap(finalDependencies0) /** * The "next" dependency set, made of the current dependencies and their transitive dependencies, @@ -397,7 +363,7 @@ case class Resolution(rootDependencies: Set[Dependency], * Returns a tuple made of the conflicting dependencies, and all the dependencies. */ def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = { - merge(rootDependencies ++ dependencies ++ transitiveDependencies) + merge(rootDependencies.map(withDefaultScope) ++ dependencies ++ transitiveDependencies) } /** @@ -456,7 +422,7 @@ case class Resolution(rootDependencies: Set[Dependency], * The versions of all the dependencies returned are erased (emptied). */ def remainingDependencies: Set[Dependency] = { - val rootDependencies0 = rootDependencies.map(eraseVersion) + val rootDependencies0 = rootDependencies.map(withDefaultScope).map(eraseVersion) @tailrec def helper(reverseDeps: Map[Dependency, Vector[Dependency]]): Map[Dependency, Vector[Dependency]] = { @@ -632,4 +598,14 @@ case class Resolution(rootDependencies: Set[Dependency], } } + def last(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], maxIterations: Int = -1): Task[Resolution] = { + if (maxIterations == 0 || isDone) Task.now(this) + else { + next(fetchModule) + .flatMap(_.last(fetchModule, if (maxIterations > 0) maxIterations - 1 else maxIterations)) + } + } + + def minDependencies: Set[Dependency] = + Orders.minDependencies(dependencies) } diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 5a9a0d079..476b730ee 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -88,6 +88,14 @@ package object coursier { maxIterations: Option[Int] = Some(200), filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { - core.Resolution.resolve(dependencies, fetch, maxIterations, filter, profileActivation) + + val startResolution = Resolution( + dependencies, Set.empty, Set.empty, + Map.empty, Map.empty, + filter, + profileActivation + ) + + startResolution.last(fetch, maxIterations.getOrElse(-1)) } } diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 9e7683f82..693769100 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -38,7 +38,7 @@ object CentralTests extends TestSuite { .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set( dep.withCompileScope, Dependency(Module("ch.qos.logback", "logback-core"), "1.1.3").withCompileScope, @@ -54,7 +54,7 @@ object CentralTests extends TestSuite { .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set( dep.withCompileScope, Dependency(Module("org.ow2.asm", "asm-tree"), "5.0.2").withCompileScope, @@ -70,7 +70,7 @@ object CentralTests extends TestSuite { val res = res0.copy(projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set( dep.withCompileScope)) diff --git a/core/src/test/scala/coursier/test/ResolutionTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala index 9e9b577b3..d10db84e5 100644 --- a/core/src/test/scala/coursier/test/ResolutionTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -166,7 +166,7 @@ object ResolutionTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), errors = Map(dep.moduleVersion -> Seq("Not found")) ) @@ -183,7 +183,7 @@ object ResolutionTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), projectsCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) ) @@ -201,7 +201,7 @@ object ResolutionTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope, trDep.withCompileScope), projectsCache = Map( projectsMap(dep.moduleVersion).kv, @@ -225,7 +225,7 @@ object ResolutionTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), projectsCache = Map( projectsMap(dep.moduleVersion).kv @@ -250,7 +250,7 @@ object ResolutionTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), projectsCache = Map( projectsMap(dep.moduleVersion).kv @@ -275,7 +275,7 @@ object ResolutionTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), projectsCache = Map( projectsMap(dep.moduleVersion).kv @@ -295,7 +295,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), projectsCache = Map( projectsMap(dep.moduleVersion).kv @@ -319,7 +319,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -339,7 +339,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -358,7 +358,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -375,7 +375,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ) @@ -394,7 +394,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -415,7 +415,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -438,7 +438,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -460,7 +460,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -484,7 +484,7 @@ object ResolutionTests extends TestSuite { ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) val expected = Resolution( - rootDependencies = deps.map(_.withCompileScope), + rootDependencies = deps, dependencies = (deps ++ trDeps).map(_.withCompileScope) ) diff --git a/core/src/test/scala/coursier/test/package.scala b/core/src/test/scala/coursier/test/package.scala index 863eb2519..af3c811e8 100644 --- a/core/src/test/scala/coursier/test/package.scala +++ b/core/src/test/scala/coursier/test/package.scala @@ -1,9 +1,26 @@ package coursier +import scalaz.EitherT +import scalaz.concurrent.Task + package object test { implicit class DependencyOps(val underlying: Dependency) extends AnyVal { def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile) } + def resolve(dependencies: Set[Dependency], + fetch: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], + maxIterations: Option[Int] = Some(200), + filter: Option[Dependency => Boolean] = None, + profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { + + val startResolution = Resolution( + dependencies, + filter = filter, + profileActivation = profileActivation + ) + + startResolution.last(fetch, maxIterations.getOrElse(-1)) + } } diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index c37f491e0..317ec5efe 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,7 +1,7 @@ package coursier package web -import coursier.core.{Resolver, Logger, Remote} +import coursier.core.{Logger, Remote} import japgolly.scalajs.react.vdom.{TagMod, Attr} import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope} @@ -71,13 +71,15 @@ class Backend($: BackendScope[Unit, State]) { def updateTree(resolution: Resolution, target: String, reverse: Boolean) = { def depsOf(dep: Dependency) = - resolution.projectsCache.get(dep.moduleVersion).toSeq.flatMap(t => Resolver.finalDependencies(dep, t._2).filter(resolution.filter getOrElse Resolver.defaultFilter)) + resolution.projectsCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter)) + + val minDependencies = resolution.minDependencies lazy val reverseDeps = { var m = Map.empty[Module, Seq[Dependency]] for { - dep <- resolution.dependencies + dep <- minDependencies trDep <- depsOf(dep) } { m += trDep.module -> (m.getOrElse(trDep.module, Nil) :+ dep) @@ -95,8 +97,8 @@ class Backend($: BackendScope[Unit, State]) { else Seq("nodes" -> js.Array(deps.map(tree): _*)) }: _*) - println(resolution.dependencies.toList.map(tree).map(js.JSON.stringify(_))) - g.$(target).treeview(js.Dictionary("data" -> js.Array(resolution.dependencies.toList.map(tree): _*))) + println(minDependencies.toList.map(tree).map(js.JSON.stringify(_))) + g.$(target).treeview(js.Dictionary("data" -> js.Array(minDependencies.toList.map(tree): _*))) } def resolve(action: => Unit = ()) = { @@ -119,11 +121,14 @@ class Backend($: BackendScope[Unit, State]) { } val s = $.state - def task = coursier.resolve( - s.modules.toSet, - fetchFrom(s.repositories.map(_.copy(logger = Some(logger)))), - filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) - ) + def task = { + val res = coursier.Resolution( + s.modules.toSet, + filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) + ) + + res.last(fetchFrom(s.repositories.map(_.copy(logger = Some(logger)))), 100) + } // For reasons that are unclear to me, not delaying this when using the runNow execution context // somehow discards the $.modState above. (Not a major problem as queue is used by default.) @@ -258,7 +263,7 @@ object App { ) } - val sortedDeps = res.dependencies.toList + val sortedDeps = res.minDependencies.toList .sortBy(dep => coursier.core.Module.unapply(dep.module).get) <.table(^.`class` := "table", From 07278edd6da4fd7d57ce10425c1cea32a1663762 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:51 +0100 Subject: [PATCH 06/12] Add comment --- core/src/main/scala/coursier/core/Orders.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/scala/coursier/core/Orders.scala b/core/src/main/scala/coursier/core/Orders.scala index 1653e99d7..569596bae 100644 --- a/core/src/main/scala/coursier/core/Orders.scala +++ b/core/src/main/scala/coursier/core/Orders.scala @@ -107,6 +107,13 @@ object Orders { .groupBy(dep => (dep.optional, dep.scope)) .toList + /* + * Iterates over all pairs (xDep, yDep) from `dependencies`. + * If xDep < yDep (all that yDep brings is already brought by xDep), remove yDep. + * + * The (partial) order on dependencies is made of the ones on scope, optional, and exclusions. + */ + val remove = for { List(((xOpt, xScope), xDeps), ((yOpt, yScope), yDeps)) <- groupedDependencies.combinations(2) From 5dc17a5bb380f88901ec1d85ef62b77fa6b408f9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:52 +0100 Subject: [PATCH 07/12] Better minification --- .../main/scala/coursier/core/Exclusions.scala | 38 ++++++++++++++++++- .../src/main/scala/coursier/core/Orders.scala | 14 ++----- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/coursier/core/Exclusions.scala b/core/src/main/scala/coursier/core/Exclusions.scala index 742961223..997c9bc61 100644 --- a/core/src/main/scala/coursier/core/Exclusions.scala +++ b/core/src/main/scala/coursier/core/Exclusions.scala @@ -8,7 +8,7 @@ object Exclusions { .partition{case (org, name) => org == "*" || name == "*" } val all = wildCards - .contains(("*", "*")) + .contains(one.head) val excludeByOrg = wildCards .collect{case (org, "*") if org != "*" => org } @@ -35,7 +35,7 @@ object Exclusions { val (all, excludeByOrg, excludeByName, remaining) = partition(exclusions) - if (all) Set(("*", "*")) + if (all) one else { val filteredRemaining = remaining .filter{case (org, name) => @@ -49,5 +49,39 @@ object Exclusions { } } + val zero = Set.empty[(String, String)] + val one = Set(("*", "*")) + + def join(x: Set[(String, String)], y: Set[(String, String)]): Set[(String, String)] = + minimize(x ++ y) + + def meet(x: Set[(String, String)], y: Set[(String, String)]): Set[(String, String)] = { + + val ((xAll, xExcludeByOrg, xExcludeByName, xRemaining), (yAll, yExcludeByOrg, yExcludeByName, yRemaining)) = + (partition(x), partition(y)) + + val all = xAll && yAll + + if (all) one + else { + val excludeByOrg = + if (xAll) yExcludeByOrg + else if (yAll) xExcludeByOrg + else xExcludeByOrg intersect yExcludeByOrg + val excludeByName = + if (xAll) yExcludeByName + else if (yAll) xExcludeByName + else xExcludeByName intersect yExcludeByName + + val remaining = + xRemaining.filter{case (org, name) => yAll || yExcludeByOrg(org) || yExcludeByName(name)} ++ + yRemaining.filter{case (org, name) => xAll || xExcludeByOrg(org) || xExcludeByName(name)} ++ + (xRemaining intersect yRemaining) + + excludeByOrg.map((_, "*")) ++ + excludeByName.map(("*", _)) ++ + remaining + } + } } diff --git a/core/src/main/scala/coursier/core/Orders.scala b/core/src/main/scala/coursier/core/Orders.scala index 569596bae..dd5c7a28a 100644 --- a/core/src/main/scala/coursier/core/Orders.scala +++ b/core/src/main/scala/coursier/core/Orders.scala @@ -105,23 +105,15 @@ object Orders { def minDependenciesUnsafe(dependencies: Set[Dependency]): Set[Dependency] = { val groupedDependencies = dependencies .groupBy(dep => (dep.optional, dep.scope)) + .mapValues(deps => deps.head.copy(exclusions = deps.foldLeft(Exclusions.one)((acc, dep) => Exclusions.meet(acc, dep.exclusions)))) .toList - /* - * Iterates over all pairs (xDep, yDep) from `dependencies`. - * If xDep < yDep (all that yDep brings is already brought by xDep), remove yDep. - * - * The (partial) order on dependencies is made of the ones on scope, optional, and exclusions. - */ - val remove = for { - List(((xOpt, xScope), xDeps), ((yOpt, yScope), yDeps)) <- groupedDependencies.combinations(2) + List(((xOpt, xScope), xDep), ((yOpt, yScope), yDep)) <- groupedDependencies.combinations(2) optCmp <- optionalPartialOrder.cmp(xOpt, yOpt).iterator scopeCmp <- mavenScopePartialOrder.cmp(xScope, yScope).iterator if optCmp*scopeCmp >= 0 - xDep <- xDeps.iterator - yDep <- yDeps.iterator exclCmp <- exclusionsPartialOrder.cmp(xDep.exclusions, yDep.exclusions).iterator if optCmp*exclCmp >= 0 if scopeCmp*exclCmp >= 0 @@ -130,7 +122,7 @@ object Orders { if xIsMin || yIsMin // should be always true, unless xDep == yDep, which shouldn't happen } yield if (xIsMin) yDep else xDep - dependencies -- remove + groupedDependencies.map(_._2).toSet -- remove } /** From d2885c59b49fc55235986f607321160db869258d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:53 +0100 Subject: [PATCH 08/12] Add stream method to Resolution --- core/src/main/scala/coursier/core/Resolution.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index f03f5ad67..2b4805daf 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -606,6 +606,13 @@ case class Resolution(rootDependencies: Set[Dependency], } } + def stream(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], run: Task[Resolution] => Resolution): Stream[Resolution] = { + this #:: { + if (isDone) Stream.empty + else run(next(fetchModule)).stream(fetchModule, run) + } + } + def minDependencies: Set[Dependency] = Orders.minDependencies(dependencies) } From d30be245dcf14a404192ff33ac6f8b6147ce2125 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:54 +0100 Subject: [PATCH 09/12] Rename MavenArtifact to Attributes --- cli/src/main/scala/coursier/cli/Coursier.scala | 2 +- .../src/main/scala/coursier/core/Remote.scala | 6 +++--- .../main/scala/coursier/core/Definitions.scala | 8 +++----- .../main/scala/coursier/core/Resolution.scala | 8 ++++---- core/src/main/scala/coursier/core/Xml.scala | 2 +- core/src/main/scala/coursier/package.scala | 16 ++++++++-------- .../test/scala/coursier/test/CentralTests.scala | 2 +- .../scala/coursier/test/PomParsingTests.scala | 2 +- web/src/main/scala/coursier/web/Backend.scala | 4 ++-- 9 files changed, 24 insertions(+), 26 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 23f7cb2de..a131660e2 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -104,7 +104,7 @@ case class Coursier(scope: List[String], if (version == dep.version) "" else s" ($version for ${dep.version})" - s"${dep.module.organization}:${dep.module.name}:${dep.artifact.`type`}:${Some(dep.artifact.classifier).filter(_.nonEmpty).map(_+":").mkString}$version$extra" + s"${dep.module.organization}:${dep.module.name}:${dep.attributes.`type`}:${Some(dep.attributes.classifier).filter(_.nonEmpty).map(_+":").mkString}$version$extra" } val trDeps = res.minDependencies.toList.sortBy(repr) diff --git a/core-jvm/src/main/scala/coursier/core/Remote.scala b/core-jvm/src/main/scala/coursier/core/Remote.scala index 414a8a00f..27f0427e1 100644 --- a/core-jvm/src/main/scala/coursier/core/Remote.scala +++ b/core-jvm/src/main/scala/coursier/core/Remote.scala @@ -21,14 +21,14 @@ case class ArtifactDownloader(root: String, cache: File, logger: Option[Artifact def artifact(module: Module, version: String, - artifact: Dependency.MavenArtifact, + attributes: Attributes, cachePolicy: CachePolicy): EitherT[Task, String, File] = { val relPath = module.organization.split('.').toSeq ++ Seq( module.name, version, - s"${module.name}-$version${Some(artifact.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${artifact.`type`}" + s"${module.name}-$version${Some(attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${attributes.`type`}" ) val file = (cache /: relPath)(new File(_, _)) @@ -96,7 +96,7 @@ case class ArtifactDownloader(root: String, cache: File, logger: Option[Artifact cachePolicy: CachePolicy = CachePolicy.Default): Task[Seq[String \/ File]] = { // Important: using version from project, as the one from dependency can be an interval - artifact(dependency.module, project.version, dependency.artifact, cachePolicy = cachePolicy).run.map(Seq(_)) + artifact(dependency.module, project.version, dependency.attributes, cachePolicy = cachePolicy).run.map(Seq(_)) } } diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index b0836f7f7..6a331e291 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -32,16 +32,14 @@ sealed abstract class Scope(val name: String) case class Dependency(module: Module, version: String, scope: Scope, - artifact: Dependency.MavenArtifact, + attributes: Attributes, exclusions: Set[(String, String)], optional: Boolean) { def moduleVersion = (module, version) } -object Dependency { - case class MavenArtifact(`type`: String, - classifier: String) -} +case class Attributes(`type`: String, + classifier: String) case class Project(module: Module, version: String, diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index 2b4805daf..8596bb480 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -70,7 +70,7 @@ object Resolution { type DepMgmtKey = (String, String, String) def dependencyManagementKey(dep: Dependency): DepMgmtKey = - (dep.module.organization, dep.module.name, dep.artifact.`type`) + (dep.module.organization, dep.module.name, dep.attributes.`type`) def dependencyManagementAdd(m: Map[DepMgmtKey, Dependency], dep: Dependency): Map[DepMgmtKey, Dependency] = { val key = dependencyManagementKey(dep) if (m.contains(key)) m else m + (key -> dep) @@ -119,9 +119,9 @@ object Resolution { name = substituteProps(dep.module.name) ), version = substituteProps(dep.version), - artifact = dep.artifact.copy( - `type` = substituteProps(dep.artifact.`type`), - classifier = substituteProps(dep.artifact.classifier) + attributes = dep.attributes.copy( + `type` = substituteProps(dep.attributes.`type`), + classifier = substituteProps(dep.attributes.classifier) ), scope = Parse.scope(substituteProps(dep.scope.name)), exclusions = dep.exclusions diff --git a/core/src/main/scala/coursier/core/Xml.scala b/core/src/main/scala/coursier/core/Xml.scala index 2c92a03bc..caa2a3156 100644 --- a/core/src/main/scala/coursier/core/Xml.scala +++ b/core/src/main/scala/coursier/core/Xml.scala @@ -85,7 +85,7 @@ object Xml { mod, version0, scopeOpt getOrElse defaultScope, - Dependency.MavenArtifact(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), + Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), exclusions.map(mod => (mod.organization, mod.name)).toSet, optional ) diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 476b730ee..87402dff0 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -11,17 +11,17 @@ package object coursier { def apply(module: Module, version: String, scope: Scope = Scope.Other(""), // Substituted by Resolver with its own default scope (compile) - artifact: MavenArtifact = MavenArtifact(), + attributes: Attributes = Attributes(), exclusions: Set[(String, String)] = Set.empty, optional: Boolean = false): Dependency = - core.Dependency(module, version, scope, artifact, exclusions, optional) + core.Dependency(module, version, scope, attributes, exclusions, optional) + } - type MavenArtifact = core.Dependency.MavenArtifact - object MavenArtifact { - def apply(`type`: String = "jar", - classifier: String = ""): MavenArtifact = - core.Dependency.MavenArtifact(`type`, classifier) - } + type Attributes = core.Attributes + object Attributes { + def apply(`type`: String = "jar", + classifier: String = ""): Attributes = + core.Attributes(`type`, classifier) } type Project = core.Project diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 693769100..3b36d9694 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -13,7 +13,7 @@ object CentralTests extends TestSuite { ) def repr(dep: Dependency) = - s"${dep.module.organization}:${dep.module.name}:${dep.artifact.`type`}:${Some(dep.artifact.classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" + s"${dep.module.organization}:${dep.module.name}:${dep.attributes.`type`}:${Some(dep.attributes.classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" def resolutionCheck(module: Module, version: String) = async { diff --git a/core/src/test/scala/coursier/test/PomParsingTests.scala b/core/src/test/scala/coursier/test/PomParsingTests.scala index 8b3d9fbf9..3135c4905 100644 --- a/core/src/test/scala/coursier/test/PomParsingTests.scala +++ b/core/src/test/scala/coursier/test/PomParsingTests.scala @@ -21,7 +21,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Dependency(Module("comp", "lib"), "2.1", artifact = Dependency.MavenArtifact(classifier = "extra"))) + val expected = \/-(Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra"))) val result = Xml.dependency(xmlParse(depNode).right.get) diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 317ec5efe..3cad52b95 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -230,8 +230,8 @@ object App { <.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")), <.td(Seq[Seq[TagMod]]( if (dep.scope == Scope.Compile) Seq() else Seq(infoLabel(dep.scope.name)), - if (dep.artifact.`type`.isEmpty || dep.artifact.`type` == "jar") Seq() else Seq(infoLabel(dep.artifact.`type`)), - if (dep.artifact.classifier.isEmpty) Seq() else Seq(infoLabel(dep.artifact.classifier)), + if (dep.attributes.`type`.isEmpty || dep.attributes.`type` == "jar") Seq() else Seq(infoLabel(dep.attributes.`type`)), + if (dep.attributes.classifier.isEmpty) Seq() else Seq(infoLabel(dep.attributes.classifier)), Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq, if (dep.optional) Seq(infoLabel("optional")) else Seq(), res.errors.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq From 041d0c4fb4f45174cf9cfb7160ce7a949d9daded Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:55 +0100 Subject: [PATCH 10/12] Fix --- core-jvm/src/main/scala/coursier/core/Remote.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-jvm/src/main/scala/coursier/core/Remote.scala b/core-jvm/src/main/scala/coursier/core/Remote.scala index 27f0427e1..e584e9347 100644 --- a/core-jvm/src/main/scala/coursier/core/Remote.scala +++ b/core-jvm/src/main/scala/coursier/core/Remote.scala @@ -130,7 +130,7 @@ object Remote { \/.fromTryCatchNonFatal { val is0 = is val b = - try readFullySync(is) + try readFullySync(is0) finally is0.close() new String(b, "UTF-8") From 9fd56848a8c2978a6b71325df5dc50a53b214f18 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:57 +0100 Subject: [PATCH 11/12] Refactor artifact API --- .../main/scala/coursier/cli/Coursier.scala | 76 ++---- .../coursier/core/DefaultFetchMetadata.scala | 94 +++++++ .../src/main/scala/coursier/core/Remote.scala | 138 ---------- .../coursier/core/compatibility/package.scala | 4 + .../scala/coursier/repository/package.scala | 11 +- .../test/scala/coursier/test/JsTests.scala | 5 +- .../coursier/core/DefaultFetchMetadata.scala | 101 ++++++++ .../src/main/scala/coursier/core/Remote.scala | 243 ------------------ .../coursier/core/compatibility/package.scala | 2 + .../scala/coursier/repository/package.scala | 11 +- .../coursier/test/compatibility/package.scala | 4 +- .../scala/coursier/core/Definitions.scala | 25 ++ .../main/scala/coursier/core/Repository.scala | 196 ++++++++++++-- .../main/scala/coursier/core/Resolution.scala | 61 +++-- core/src/main/scala/coursier/package.scala | 18 +- .../scala/coursier/test/CentralTests.scala | 10 +- .../scala/coursier/test/ResolutionTests.scala | 32 +-- .../scala/coursier/test/TestRepository.scala | 5 +- files/src/main/scala/coursier/Files.scala | 121 +++++++++ project/Coursier.scala | 14 +- web/src/main/scala/coursier/web/Backend.scala | 32 +-- 21 files changed, 665 insertions(+), 538 deletions(-) create mode 100644 core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala delete mode 100644 core-js/src/main/scala/coursier/core/Remote.scala create mode 100644 core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala delete mode 100644 core-jvm/src/main/scala/coursier/core/Remote.scala create mode 100644 files/src/main/scala/coursier/Files.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index a131660e2..3f06b9891 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -5,10 +5,9 @@ import java.io.File import caseapp._ import coursier.core.{CachePolicy, Parse} -import coursier.core.{ArtifactDownloaderLogger, RemoteLogger, ArtifactDownloader} +import coursier.core.MetadataFetchLogger import scalaz.concurrent.Task -import scalaz.{-\/, \/-} case class Coursier(scope: List[String], keepOptional: Boolean, @@ -20,13 +19,14 @@ case class Coursier(scope: List[String], else scope.map(Parse.scope) val scopes = scopes0.toSet - val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/central") + val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/metadata/central") + val centralFilesCacheDir = new File(sys.props("user.home") + "/.coursier/cache/files/central") val base = centralCacheDir.toURI def fileRepr(f: File) = base.relativize(f.toURI).getPath - val logger: RemoteLogger with ArtifactDownloaderLogger = new RemoteLogger with ArtifactDownloaderLogger { + val logger: MetadataFetchLogger with FilesLogger = new MetadataFetchLogger with FilesLogger { def println(s: String) = Console.err.println(s) def downloading(url: String) = @@ -53,35 +53,36 @@ case class Coursier(scope: List[String], ) } - val cachedMavenCentral = repository.mavenCentral.copy(cache = Some(centralCacheDir), logger = Some(logger)) + val cachedMavenCentral = repository.mavenCentral.copy( + fetchMetadata = repository.mavenCentral.fetchMetadata.copy( + cache = Some(centralCacheDir), + logger = Some(logger) + ) + ) val repositories = Seq[Repository]( cachedMavenCentral ) - lazy val downloaders = Map[Repository, ArtifactDownloader]( - cachedMavenCentral -> ArtifactDownloader(repository.mavenCentral.root, centralCacheDir, logger = Some(logger)) - ) - - val (splitArtifacts, malformed) = remainingArgs.toList + val (splitDependencies, malformed) = remainingArgs.toList .map(_.split(":", 3).toSeq) .partition(_.length == 3) - if (splitArtifacts.isEmpty) { - Console.err.println("Usage: coursier artifacts...") + if (splitDependencies.isEmpty) { + Console.err.println("Usage: coursier dependencies...") sys exit 1 } if (malformed.nonEmpty) { - Console.err.println(s"Malformed artifacts:\n${malformed.map(_.mkString(":")).mkString("\n")}") + Console.err.println(s"Malformed dependencies:\n${malformed.map(_.mkString(":")).mkString("\n")}") sys exit 1 } - val modules = splitArtifacts.map{ + val moduleVersions = splitDependencies.map{ case Seq(org, name, version) => (Module(org, name), version) } - val deps = modules.map{case (mod, ver) => + val deps = moduleVersions.map{case (mod, ver) => Dependency(mod, ver, scope = Scope.Runtime) } @@ -99,7 +100,7 @@ case class Coursier(scope: List[String], def repr(dep: Dependency) = { // dep.version can be an interval, whereas the one from project can't - val version = res.projectsCache.get(dep.moduleVersion).map(_._2.version).getOrElse(dep.version) + val version = res.projectCache.get(dep.moduleVersion).map(_._2.version).getOrElse(dep.version) val extra = if (version == dep.version) "" else s" ($version for ${dep.version})" @@ -116,11 +117,11 @@ case class Coursier(scope: List[String], println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") } - val errDeps = trDeps.filter(dep => res.errors.contains(dep.moduleVersion)) - if (errDeps.nonEmpty) { - println(s"${errDeps.size} error(s):") - for (dep <- errDeps) { - println(s" ${dep.module}:\n ${res.errors(dep.moduleVersion).mkString("\n").replace("\n", " \n")}") + val errors = res.errors + if (errors.nonEmpty) { + println(s"${errors.size} error(s):") + for ((dep, errs) <- errors) { + println(s" ${dep.module}:\n ${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") } } @@ -129,38 +130,11 @@ case class Coursier(scope: List[String], val cachePolicy: CachePolicy = CachePolicy.Default - val m = res.minDependencies.groupBy(dep => res.projectsCache.get(dep.moduleVersion).map(_._1)) - val (notFound, remaining0) = m.partition(_._1.isEmpty) - if (notFound.nonEmpty) { - val notFound0 = notFound.values.flatten.toList.map(repr).sorted - println(s"Not found:${notFound0.mkString("\n")}") - } + val artifacts = res.artifacts - val (remaining, downloaderNotFound) = remaining0.partition(t => downloaders.contains(t._1.get)) - if (downloaderNotFound.nonEmpty) { - val downloaderNotFound0 = downloaderNotFound.values.flatten.toList.map(repr).sorted - println(s"Don't know how to download:${downloaderNotFound0.mkString("\n")}") - } - - val sorted = remaining - .toList - .map{ case (Some(repo), deps) => repo -> deps.toList.sortBy(repr) } - .sortBy(_._1.toString) // ... - - val tasks = - for { - (repo, deps) <- sorted - dl = downloaders(repo) - dep <- deps - (_, proj) = res.projectsCache(dep.moduleVersion) - } yield { - dl.artifacts(dep, proj, cachePolicy = cachePolicy).map { results => - val errorCount = results.count{case -\/(_) => true; case _ => false} - val resultsRepr = results.map(_.map(fileRepr).merge).map(" " + _).mkString("\n") - println(s"${repr(dep)} (${results.length} artifact(s)${if (errorCount > 0) s", $errorCount error(s)" else ""}):\n$resultsRepr") - } - } + val files = new Files(Seq(cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir), () => ???, Some(logger)) + val tasks = artifacts.map(files.file(_, cachePolicy).run) val task = Task.gatherUnordered(tasks) task.run diff --git a/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala new file mode 100644 index 000000000..d41b995bf --- /dev/null +++ b/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala @@ -0,0 +1,94 @@ +package coursier +package core + +import org.scalajs.dom.raw.{Event, XMLHttpRequest} + +import scala.concurrent.{ExecutionContext, Promise, Future} +import scalaz.{-\/, \/-, EitherT} +import scalaz.concurrent.Task + +import scala.scalajs.js +import js.Dynamic.{global => g} + +import scala.scalajs.js.timers._ + +object DefaultFetchMetadata { + + def encodeURIComponent(s: String): String = + g.encodeURIComponent(s).asInstanceOf[String] + + lazy val jsonpAvailable = !js.isUndefined(g.jsonp) + + /** Available if we're running on node, and package xhr2 is installed */ + lazy val xhr = g.require("xhr2") + def xhrReq() = + js.Dynamic.newInstance(xhr)().asInstanceOf[XMLHttpRequest] + + def fetchTimeout(target: String, p: Promise[_]) = + setTimeout(5000) { + if (!p.isCompleted) { + p.failure(new Exception(s"Timeout when fetching $target")) + } + } + + // FIXME Take into account HTTP error codes from YQL response + def proxiedJsonp(url: String)(implicit executionContext: ExecutionContext): Future[String] = { + val url0 = + "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20xml%20where%20url%3D%22" + + encodeURIComponent(url) + + "%22&format=jsonp&diagnostics=true" + + val p = Promise[String]() + + g.jsonp(url0, (res: js.Dynamic) => if (!p.isCompleted) { + val success = !js.isUndefined(res) && !js.isUndefined(res.results) + if (success) + p.success(res.results.asInstanceOf[js.Array[String]].mkString("\n")) + else + p.failure(new Exception(s"Fetching $url ($url0)")) + }) + + fetchTimeout(s"$url ($url0)", p) + p.future + } + + def get(url: String)(implicit executionContext: ExecutionContext): Future[String] = + if (jsonpAvailable) + proxiedJsonp(url) + else { + val p = Promise[String]() + val xhrReq0 = xhrReq() + val f = { _: Event => + p.success(xhrReq0.responseText) + } + xhrReq0.onload = f + + xhrReq0.open("GET", url) + xhrReq0.send() + + fetchTimeout(url, p) + p.future + } + +} + +trait Logger { + def fetching(url: String): Unit + def fetched(url: String): Unit + def other(url: String, msg: String): Unit +} + +case class DefaultFetchMetadata(root: String, + logger: Option[Logger] = None) extends FetchMetadata { + def apply(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, String] = { + + EitherT( + Task { implicit ec => + DefaultFetchMetadata.get(root + artifact.url) + .map(\/-(_)) + .recover{case e: Exception => -\/(e.getMessage)} + } + ) + } +} \ No newline at end of file diff --git a/core-js/src/main/scala/coursier/core/Remote.scala b/core-js/src/main/scala/coursier/core/Remote.scala deleted file mode 100644 index 6aa2d2700..000000000 --- a/core-js/src/main/scala/coursier/core/Remote.scala +++ /dev/null @@ -1,138 +0,0 @@ -package coursier -package core - -import org.scalajs.dom.raw.{Event, XMLHttpRequest} - -import scala.concurrent.{ExecutionContext, Promise, Future} -import scalaz.{-\/, \/-, \/, EitherT} -import scalaz.concurrent.Task - -import scala.scalajs.js -import js.Dynamic.{global => g} - -import scala.scalajs.js.timers._ - -object Remote { - - def encodeURIComponent(s: String): String = - g.encodeURIComponent(s).asInstanceOf[String] - - lazy val jsonpAvailable = !js.isUndefined(g.jsonp) - - /** Available if we're running on node, and package xhr2 is installed */ - lazy val xhr = g.require("xhr2") - def xhrReq() = - js.Dynamic.newInstance(xhr)().asInstanceOf[XMLHttpRequest] - - def fetchTimeout(target: String, p: Promise[_]) = - setTimeout(5000) { - if (!p.isCompleted) { - p.failure(new Exception(s"Timeout when fetching $target")) - } - } - - def proxiedJsonp(url: String)(implicit executionContext: ExecutionContext): Future[String] = { - val url0 = - "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20xml%20where%20url%3D%22" + - encodeURIComponent(url) + - "%22&format=jsonp&diagnostics=true" - - val p = Promise[String]() - - g.jsonp(url0, (res: js.Dynamic) => if (!p.isCompleted) { - val success = !js.isUndefined(res) && !js.isUndefined(res.results) - if (success) - p.success(res.results.asInstanceOf[js.Array[String]].mkString("\n")) - else - p.failure(new Exception(s"Fetching $url ($url0)")) - }) - - fetchTimeout(s"$url ($url0)", p) - p.future - } - - def get(url: String)(implicit executionContext: ExecutionContext): Future[Either[String, Xml.Node]] = - if (jsonpAvailable) - proxiedJsonp(url).map(compatibility.xmlParse) - else { - val p = Promise[Either[String, Xml.Node]]() - val xhrReq0 = xhrReq() - val f = { _: Event => - p.success(compatibility.xmlParse(xhrReq0.responseText)) - } - xhrReq0.onload = f - - xhrReq0.open("GET", url) - xhrReq0.send() - - fetchTimeout(url, p) - p.future - } - -} - -trait Logger { - def fetching(url: String): Unit - def fetched(url: String): Unit - def other(url: String, msg: String): Unit -} - -case class Remote(base: String, logger: Option[Logger] = None) extends MavenRepository { - - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { - - val path = { - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version.pom" - ) - } .map(Remote.encodeURIComponent) - - val url = base + path.mkString("/") - - EitherT(Task{ implicit ec => - logger.foreach(_.fetching(url)) - Remote.get(url).recover{case e: Exception => Left(e.getMessage)}.map{ eitherXml => - logger.foreach(_.fetched(url)) - for { - xml <- \/.fromEither(eitherXml) - _ = logger.foreach(_.other(url, "is XML")) - _ <- if (xml.label == "project") \/-(()) else -\/(s"Project definition not found (got '${xml.label}')") - _ = logger.foreach(_.other(url, "project definition found")) - proj <- Xml.project(xml) - _ = logger.foreach(_.other(url, "project definition ok")) - } yield proj - } - }) - } - - def versions(organization: String, - name: String, - cachePolicy: CachePolicy): EitherT[Task, String, Versions] = { - - val path = { - organization.split('.').toSeq ++ Seq( - name, - "maven-metadata.xml" - ) - } .map(Remote.encodeURIComponent) - - val url = base + path.mkString("/") - - EitherT(Task{ implicit ec => - logger.foreach(_.fetching(url)) - Remote.get(url).recover{case e: Exception => Left(e.getMessage)}.map{ eitherXml => - logger.foreach(_.fetched(url)) - for { - xml <- \/.fromEither(eitherXml) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - versions <- Xml.versions(xml) - } yield versions - } - }) - } - -} diff --git a/core-js/src/main/scala/coursier/core/compatibility/package.scala b/core-js/src/main/scala/coursier/core/compatibility/package.scala index 4c57573d7..5c940cb9d 100644 --- a/core-js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core-js/src/main/scala/coursier/core/compatibility/package.scala @@ -1,6 +1,7 @@ package coursier.core import scala.scalajs.js +import js.Dynamic.{global => g} import org.scalajs.dom.raw.NodeList package object compatibility { @@ -94,4 +95,7 @@ package object compatibility { Right(doc.fold(Xml.Node.empty)(fromNode)) } + def encodeURIComponent(s: String): String = + g.encodeURIComponent(s).asInstanceOf[String] + } diff --git a/core-js/src/main/scala/coursier/repository/package.scala b/core-js/src/main/scala/coursier/repository/package.scala index 59d758a39..c448f25c5 100644 --- a/core-js/src/main/scala/coursier/repository/package.scala +++ b/core-js/src/main/scala/coursier/repository/package.scala @@ -1,13 +1,12 @@ package coursier +import coursier.core.DefaultFetchMetadata + package object repository { - type Remote = core.Remote - val Remote: core.Remote.type = core.Remote + val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/")) - val mavenCentral = Remote("https://repo1.maven.org/maven2/") - - val sonatypeReleases = Remote("https://oss.sonatype.org/content/repositories/releases/") - val sonatypeSnapshots = Remote("https://oss.sonatype.org/content/repositories/snapshots/") + val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) + val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) } diff --git a/core-js/src/test/scala/coursier/test/JsTests.scala b/core-js/src/test/scala/coursier/test/JsTests.scala index 5888d9331..9738b2600 100644 --- a/core-js/src/test/scala/coursier/test/JsTests.scala +++ b/core-js/src/test/scala/coursier/test/JsTests.scala @@ -1,7 +1,7 @@ package coursier package test -import coursier.core.Remote +import coursier.core.DefaultFetchMetadata import coursier.test.compatibility._ import utest._ @@ -18,7 +18,8 @@ object JsTests extends TestSuite { } 'get{ - Remote.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + DefaultFetchMetadata.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + .map(core.compatibility.xmlParse) .map{ xml => assert(xml.right.toOption.exists(_.label == "project")) } diff --git a/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala new file mode 100644 index 000000000..30f7f87af --- /dev/null +++ b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala @@ -0,0 +1,101 @@ +package coursier +package core + +import java.io._ +import java.net.URL + +import scala.io.Codec +import scalaz._, Scalaz._ +import scalaz.concurrent.Task + +trait MetadataFetchLogger { + def downloading(url: String): Unit + def downloaded(url: String, success: Boolean): Unit + def readingFromCache(f: File): Unit + def puttingInCache(f: File): Unit +} + +case class DefaultFetchMetadata(root: String, + cache: Option[File] = None, + logger: Option[MetadataFetchLogger] = None) extends FetchMetadata { + + def apply(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, String] = { + lazy val localFile = { + for { + cache0 <- cache.toRightDisjunction("No cache") + f = new File(cache0, artifact.url) + } yield f + } + + def locally = { + Task { + for { + f0 <- localFile + f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") + content <- \/.fromTryCatchNonFatal{ + logger.foreach(_.readingFromCache(f)) + scala.io.Source.fromFile(f)(Codec.UTF8).mkString + }.leftMap(_.getMessage) + } yield content + } + } + + def remote = { + val urlStr = root + artifact.url + val url = new URL(urlStr) + + def log = Task(logger.foreach(_.downloading(urlStr))) + def get = DefaultFetchMetadata.readFully(url.openStream()) + + log.flatMap(_ => get) + } + + def save(s: String) = { + localFile.fold(_ => Task.now(()), f => + Task { + if (!f.exists()) { + logger.foreach(_.puttingInCache(f)) + f.getParentFile.mkdirs() + val w = new PrintWriter(f) + try w.write(s) + finally w.close() + () + } + } + ) + } + + EitherT(cachePolicy.saving(locally)(remote)(save)) + } + +} + +object DefaultFetchMetadata { + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap(_.getMessage) + } + +} diff --git a/core-jvm/src/main/scala/coursier/core/Remote.scala b/core-jvm/src/main/scala/coursier/core/Remote.scala deleted file mode 100644 index e584e9347..000000000 --- a/core-jvm/src/main/scala/coursier/core/Remote.scala +++ /dev/null @@ -1,243 +0,0 @@ -package coursier -package core - -import java.io._ -import java.net.URL - -import scala.annotation.tailrec -import scala.io.Codec -import scalaz._, Scalaz._ -import scalaz.concurrent.Task - -// FIXME This kind of side-effecting API is lame, we should aim at a more functional one. -trait ArtifactDownloaderLogger { - def foundLocally(f: File): Unit - def downloadingArtifact(url: String): Unit - def downloadedArtifact(url: String, success: Boolean): Unit -} - -case class ArtifactDownloader(root: String, cache: File, logger: Option[ArtifactDownloaderLogger] = None) { - var bufferSize = 1024*1024 - - def artifact(module: Module, - version: String, - attributes: Attributes, - cachePolicy: CachePolicy): EitherT[Task, String, File] = { - - val relPath = - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version${Some(attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${attributes.`type`}" - ) - - val file = (cache /: relPath)(new File(_, _)) - - def locally = { - Task { - if (file.exists()) { - logger.foreach(_.foundLocally(file)) - \/-(file) - } - else -\/("Not found in cache") - } - } - - def remote = { - // FIXME A lot of things can go wrong here and are not properly handled: - // - checksums should be validated - // - what if the connection gets closed during the transfer (partial file on disk)? - // - what if someone is trying to write this file at the same time? (no locking of any kind yet) - // - ... - - val urlStr = root + relPath.mkString("/") - - Task { - try { - file.getParentFile.mkdirs() - - logger.foreach(_.downloadingArtifact(urlStr)) - - val url = new URL(urlStr) - val b = Array.fill[Byte](bufferSize)(0) - val in = new BufferedInputStream(url.openStream(), bufferSize) - - try { - val out = new FileOutputStream(file) - try { - @tailrec - def helper(): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - helper() - } - } - - helper() - } finally out.close() - } finally in.close() - - logger.foreach(_.downloadedArtifact(urlStr, success = true)) - \/-(file) - } - catch { case e: Exception => - logger.foreach(_.downloadedArtifact(urlStr, success = false)) - -\/(e.getMessage) - } - } - } - - EitherT(cachePolicy(locally)(remote)) - } - - def artifacts(dependency: Dependency, - project: Project, - cachePolicy: CachePolicy = CachePolicy.Default): Task[Seq[String \/ File]] = { - - // Important: using version from project, as the one from dependency can be an interval - artifact(dependency.module, project.version, dependency.attributes, cachePolicy = cachePolicy).run.map(Seq(_)) - } - -} - -// FIXME Comment of ArtifactDownloaderLogger applies here too -trait RemoteLogger { - def downloading(url: String): Unit - def downloaded(url: String, success: Boolean): Unit - def readingFromCache(f: File): Unit - def puttingInCache(f: File): Unit -} - -object Remote { - - def readFullySync(is: InputStream) = { - val buffer = new ByteArrayOutputStream() - val data = Array.ofDim[Byte](16384) - - var nRead = is.read(data, 0, data.length) - while (nRead != -1) { - buffer.write(data, 0, nRead) - nRead = is.read(data, 0, data.length) - } - - buffer.flush() - buffer.toByteArray - } - - def readFully(is: => InputStream) = - Task { - \/.fromTryCatchNonFatal { - val is0 = is - val b = - try readFullySync(is0) - finally is0.close() - - new String(b, "UTF-8") - } .leftMap(_.getMessage) - } - -} - -case class Remote(root: String, - cache: Option[File] = None, - logger: Option[RemoteLogger] = None) extends MavenRepository { - - private def get(path: Seq[String], - cachePolicy: CachePolicy): EitherT[Task, String, String] = { - - lazy val localFile = { - for { - cache0 <- cache.toRightDisjunction("No cache") - f = (cache0 /: path)(new File(_, _)) - } yield f - } - - def locally = { - Task { - for { - f0 <- localFile - f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") - content <- \/.fromTryCatchNonFatal{ - logger.foreach(_.readingFromCache(f)) - scala.io.Source.fromFile(f)(Codec.UTF8).mkString - }.leftMap(_.getMessage) - } yield content - } - } - - def remote = { - val urlStr = root + path.mkString("/") - val url = new URL(urlStr) - - def log = Task(logger.foreach(_.downloading(urlStr))) - def get = Remote.readFully(url.openStream()) - - log.flatMap(_ => get) - } - - def save(s: String) = { - localFile.fold(_ => Task.now(()), f => - Task { - if (!f.exists()) { - logger.foreach(_.puttingInCache(f)) - f.getParentFile.mkdirs() - val w = new PrintWriter(f) - try w.write(s) - finally w.close() - () - } - } - ) - } - - EitherT(cachePolicy.saving(locally)(remote)(save)) - } - - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { - - val path = - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version.pom" - ) - - val task = get(path, cachePolicy).run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") - proj <- Xml.project(xml) - } yield proj - ) - - EitherT(task) - } - - def versions(organization: String, - name: String, - cachePolicy: CachePolicy): EitherT[Task, String, Versions] = { - - val path = - organization.split('.').toSeq ++ Seq( - name, - "maven-metadata.xml" - ) - - val task = get(path, cachePolicy).run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - versions <- Xml.versions(xml) - } yield versions - ) - - EitherT(task) - } -} diff --git a/core-jvm/src/main/scala/coursier/core/compatibility/package.scala b/core-jvm/src/main/scala/coursier/core/compatibility/package.scala index 5eb8cbc97..eb4cb154c 100644 --- a/core-jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core-jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -25,4 +25,6 @@ package object compatibility { .map(fromNode) } + def encodeURIComponent(s: String): String = + new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString } diff --git a/core-jvm/src/main/scala/coursier/repository/package.scala b/core-jvm/src/main/scala/coursier/repository/package.scala index 59d758a39..c448f25c5 100644 --- a/core-jvm/src/main/scala/coursier/repository/package.scala +++ b/core-jvm/src/main/scala/coursier/repository/package.scala @@ -1,13 +1,12 @@ package coursier +import coursier.core.DefaultFetchMetadata + package object repository { - type Remote = core.Remote - val Remote: core.Remote.type = core.Remote + val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/")) - val mavenCentral = Remote("https://repo1.maven.org/maven2/") - - val sonatypeReleases = Remote("https://oss.sonatype.org/content/repositories/releases/") - val sonatypeSnapshots = Remote("https://oss.sonatype.org/content/repositories/snapshots/") + val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) + val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) } diff --git a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala index 250a46515..10c93544c 100644 --- a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala +++ b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.core.Remote +import coursier.core.DefaultFetchMetadata import scala.concurrent.{ExecutionContext, Future} import scalaz.concurrent.Task @@ -17,7 +17,7 @@ package object compatibility { def is = getClass.getClassLoader .getResource(path).openStream() - new String(Remote.readFullySync(is), "UTF-8") + new String(DefaultFetchMetadata.readFullySync(is), "UTF-8") } } diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index 6a331e291..75c775633 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -70,6 +70,7 @@ case class Profile(id: String, dependencyManagement: Seq[Dependency], properties: Map[String, String]) +// FIXME Move to MavenRepository? case class Versions(latest: String, release: String, available: List[String], @@ -78,3 +79,27 @@ case class Versions(latest: String, object Versions { case class DateTime(year: Int, month: Int, day: Int, hour: Int, minute: Int, second: Int) } + +case class Artifact(url: String, + extra: Map[String, String], + attributes: Attributes) + +object Artifact { + val md5 = "md5" + val sha1 = "sha1" + val sig = "pgp" + val sigMd5 = "md5-pgp" + val sigSha1 = "sha1-pgp" + val sources = "src" + val sourcesMd5 = "md5-src" + val sourcesSha1 = "sha1-src" + val sourcesSig = "src-pgp" + val sourcesSigMd5 = "md5-src-pgp" + val sourcesSigSha1 = "sha1-src-pgp" + val javadoc = "javadoc" + val javadocMd5 = "md5-javadoc" + val javadocSha1 = "sha1-javadoc" + val javadocSig = "javadoc-pgp" + val javadocSigMd5 = "md5-javadoc-pgp" + val javadocSigSha1 = "sha1-javadoc-pgp" +} diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index c953d64cc..d95d32eef 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -3,9 +3,11 @@ package coursier.core import scalaz.{-\/, \/-, \/, EitherT} import scalaz.concurrent.Task +import coursier.core.compatibility.encodeURIComponent + trait Repository { def find(module: Module, version: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Project] - def versions(organization: String, name: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Versions] + def artifacts(dependency: Dependency, project: Project): Seq[Artifact] } sealed trait CachePolicy { @@ -37,7 +39,132 @@ object CachePolicy { } } -trait MavenRepository extends Repository { +object Repository { + implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { + def withDefaultChecksums: Artifact = + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.md5 -> (underlying.url + ".md5"), + Artifact.sha1 -> (underlying.url + ".sha1") + )) + def withDefaultSignature: Artifact = + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.sigMd5 -> (underlying.url + ".asc.md5"), + Artifact.sigSha1 -> (underlying.url + ".asc.sha1"), + Artifact.sig -> (underlying.url + ".asc") + )) + def withJavadocSources: Artifact = { + val base = underlying.url.stripSuffix(".jar") + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.sourcesMd5 -> (base + "-sources.jar.md5"), + Artifact.sourcesSha1 -> (base + "-sources.jar.sha1"), + Artifact.sources -> (base + "-sources.jar"), + Artifact.sourcesSigMd5 -> (base + "-sources.jar.asc.md5"), + Artifact.sourcesSigSha1 -> (base + "-sources.jar.asc.sha1"), + Artifact.sourcesSig -> (base + "-sources.jar.asc"), + Artifact.javadocMd5 -> (base + "-javadoc.jar.md5"), + Artifact.javadocSha1 -> (base + "-javadoc.jar.sha1"), + Artifact.javadoc -> (base + "-javadoc.jar"), + Artifact.javadocSigMd5 -> (base + "-javadoc.jar.asc.md5"), + Artifact.javadocSigSha1 -> (base + "-javadoc.jar.asc.sha1"), + Artifact.javadocSig -> (base + "-javadoc.asc.jar") + )) + } + } +} + +trait FetchMetadata { + def root: String + def apply(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, String] +} + +case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, + ivyLike: Boolean = false) extends Repository { + + import Repository._ + + def projectArtifact(module: Module, version: String): Artifact = { + if (ivyLike) ??? + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + version, + s"${module.name}-$version.pom" + ) + ) .map(encodeURIComponent) + + Artifact( + path.mkString("/"), + Map( + Artifact.md5 -> "", + Artifact.sha1 -> "" + ), + Attributes("pom", "") + ) + .withDefaultSignature + } + } + + def versionsArtifact(module: Module): Option[Artifact] = + if (ivyLike) None + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + "maven-metadata.xml" + ) + ) .map(encodeURIComponent) + + val artifact = + Artifact( + path.mkString("/"), + Map.empty, + Attributes("pom", "") + ) + .withDefaultChecksums + + Some(artifact) + } + + def versions(module: Module, + cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Versions] = { + + EitherT( + versionsArtifact(module) match { + case None => Task.now(-\/("Not supported")) + case Some(artifact) => + fetchMetadata(artifact, cachePolicy) + .run + .map(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") + versions <- Xml.versions(xml) + } yield versions + ) + } + ) + } + + def findNoInterval(module: Module, + version: String, + cachePolicy: CachePolicy): EitherT[Task, String, Project] = { + + EitherT { + fetchMetadata(projectArtifact(module, version), cachePolicy) + .run + .map(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") + proj <- Xml.project(xml) + } yield proj + ) + } + } def find(module: Module, version: String, @@ -46,28 +173,55 @@ trait MavenRepository extends Repository { Parse.versionInterval(version).filter(_.isValid) match { case None => findNoInterval(module, version, cachePolicy) case Some(itv) => - versions(module.organization, module.name, cachePolicy).flatMap { versions0 => - val eitherVersion = { - val release = Version(versions0.release) - if (itv.contains(release)) \/-(versions0.release) - else { - val inInterval = versions0.available.map(Version(_)).filter(itv.contains) - if (inInterval.isEmpty) -\/(s"No version found for $version") - else \/-(inInterval.max.repr) + versions(module, cachePolicy) + .flatMap { versions0 => + val eitherVersion = { + val release = Version(versions0.release) + + if (itv.contains(release)) \/-(versions0.release) + else { + val inInterval = versions0.available + .map(Version(_)) + .filter(itv.contains) + + if (inInterval.isEmpty) -\/(s"No version found for $version") + else \/-(inInterval.max.repr) + } + } + + eitherVersion match { + case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason))) + case \/-(version0) => + findNoInterval(module, version0, cachePolicy) + .map(_.copy(versions = Some(versions0))) } } - - eitherVersion match { - case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason))) - case \/-(version0) => findNoInterval(module, version0, cachePolicy) - .map(_.copy(versions = Some(versions0))) - } - } } } - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] + def artifacts(dependency: Dependency, + project: Project): Seq[Artifact] = { -} + val path = + dependency.module.organization.split('.').toSeq ++ Seq( + dependency.module.name, + project.version, + s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + ) + + var artifact = + Artifact( + fetchMetadata.root + path.mkString("/"), + Map.empty, + dependency.attributes + ) + .withDefaultChecksums + + if (dependency.attributes.`type` == "jar") + artifact = artifact + .withDefaultSignature + .withJavadocSources + + Seq(artifact) + } +} \ No newline at end of file diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index 8596bb480..1f771a258 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -322,14 +322,14 @@ object Resolution { * * @param dependencies: current set of dependencies * @param conflicts: conflicting dependencies - * @param projectsCache: cache of known projects - * @param errors: keeps track of the modules whose project definition could not be found + * @param projectCache: cache of known projects + * @param errorCache: keeps track of the modules whose project definition could not be found */ case class Resolution(rootDependencies: Set[Dependency], dependencies: Set[Dependency], conflicts: Set[Dependency], - projectsCache: Map[Resolution.ModuleVersion, (Repository, Project)], - errors: Map[Resolution.ModuleVersion, Seq[String]], + projectCache: Map[Resolution.ModuleVersion, (Repository, Project)], + errorCache: Map[Resolution.ModuleVersion, Seq[String]], filter: Option[Dependency => Boolean], profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) { import Resolution._ @@ -337,7 +337,7 @@ case class Resolution(rootDependencies: Set[Dependency], private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]() private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized { finalDependenciesCache.getOrElseUpdate(dep, - projectsCache.get(dep.moduleVersion) match { + projectCache.get(dep.moduleVersion) match { case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter) case None => Nil } @@ -374,7 +374,7 @@ case class Resolution(rootDependencies: Set[Dependency], val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion) (modules ++ nextModules) - .filterNot(mod => projectsCache.contains(mod) || errors.contains(mod)) + .filterNot(mod => projectCache.contains(mod) || errorCache.contains(mod)) } @@ -477,7 +477,7 @@ case class Resolution(rootDependencies: Set[Dependency], def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = { val approxProperties = project.parent - .flatMap(projectsCache.get) + .flatMap(projectCache.get) .map(_._2.properties) .fold(project.properties)(mergeProperties(project.properties, _)) @@ -511,13 +511,13 @@ case class Resolution(rootDependencies: Set[Dependency], if (toCheck.isEmpty) missing else if (toCheck.exists(done)) helper(toCheck -- done, done, missing) else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing) - else if (toCheck.exists(projectsCache.contains)) { - val (checking, remaining) = toCheck.partition(projectsCache.contains) - val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectsCache(mod)._2)) + else if (toCheck.exists(projectCache.contains)) { + val (checking, remaining) = toCheck.partition(projectCache.contains) + val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectCache(mod)._2)) helper(remaining ++ directRequirements, done ++ checking, missing) - } else if (toCheck.exists(errors.contains)) { - val (errored, remaining) = toCheck.partition(errors.contains) + } else if (toCheck.exists(errorCache.contains)) { + val (errored, remaining) = toCheck.partition(errorCache.contains) helper(remaining, done ++ errored, missing) } else helper(Set.empty, done, missing ++ toCheck) @@ -535,8 +535,8 @@ case class Resolution(rootDependencies: Set[Dependency], val approxProperties = project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) + .filter(projectCache.contains) + .map(projectCache(_)._2.properties) .fold(project.properties)(mergeProperties(project.properties, _)) val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) @@ -546,9 +546,9 @@ case class Resolution(rootDependencies: Set[Dependency], val deps = dependencies0 - .collect{ case dep if dep.scope == Scope.Import && projectsCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ - project.parent.filter(projectsCache.contains) - val projs = deps.map(projectsCache(_)._2) + .collect{ case dep if dep.scope == Scope.Import && projectCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ + project.parent.filter(projectCache.contains) + val projs = deps.map(projectCache(_)._2) val depMgmt = (project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement))) @@ -560,13 +560,13 @@ case class Resolution(rootDependencies: Set[Dependency], dependencies = dependencies0 .filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++ project.parent - .filter(projectsCache.contains) + .filter(projectCache.contains) .toSeq - .flatMap(projectsCache(_)._2.dependencies), + .flatMap(projectCache(_)._2.dependencies), dependencyManagement = depMgmt.values.toSeq, properties = project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) + .filter(projectCache.contains) + .map(projectCache(_)._2.properties) .fold(properties0)(mergeProperties(properties0, _)) ) } @@ -580,7 +580,7 @@ case class Resolution(rootDependencies: Set[Dependency], val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) gatheredLookups.flatMap{ lookupResults => - val errors0 = errors ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} + val errors0 = errorCache ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj} /* @@ -588,12 +588,12 @@ case class Resolution(rootDependencies: Set[Dependency], * dependency management / inheritance-related bits to them. */ - newProjects.foldLeft(Task.now(copy(errors = errors0))) { case (accTask, (modVer, (repo, proj))) => + newProjects.foldLeft(Task.now(copy(errorCache = errors0))) { case (accTask, (modVer, (repo, proj))) => for { current <- accTask updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule) proj0 = updated.withDependencyManagement(proj) - } yield updated.copy(projectsCache = updated.projectsCache + (modVer -> (repo, proj0))) + } yield updated.copy(projectCache = updated.projectCache + (modVer -> (repo, proj0))) } } } @@ -615,4 +615,17 @@ case class Resolution(rootDependencies: Set[Dependency], def minDependencies: Set[Dependency] = Orders.minDependencies(dependencies) + + def artifacts: Seq[Artifact] = + for { + dep <- minDependencies.toSeq + (repo, proj) <- projectCache.get(dep.moduleVersion).toSeq + artifact <- repo.artifacts(dep, proj) + } yield artifact + + def errors: Seq[(Dependency, Seq[String])] = + for { + dep <- dependencies.toSeq + err <- errorCache.get(dep.moduleVersion).toSeq + } yield (dep, err) } diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 87402dff0..2cfd39143 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -76,11 +76,11 @@ package object coursier { def apply(rootDependencies: Set[Dependency] = Set.empty, dependencies: Set[Dependency] = Set.empty, conflicts: Set[Dependency] = Set.empty, - projectsCache: Map[ModuleVersion, (Repository, Project)] = Map.empty, - errors: Map[ModuleVersion, Seq[String]] = Map.empty, + projectCache: Map[ModuleVersion, (Repository, Project)] = Map.empty, + errorCache: Map[ModuleVersion, Seq[String]] = Map.empty, filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution = - core.Resolution(rootDependencies, dependencies, conflicts, projectsCache, errors, filter, profileActivation) + core.Resolution(rootDependencies, dependencies, conflicts, projectCache, errorCache, filter, profileActivation) } def resolve(dependencies: Set[Dependency], @@ -98,4 +98,16 @@ package object coursier { startResolution.last(fetch, maxIterations.getOrElse(-1)) } + + type Artifact = core.Artifact + object Artifact { + def apply(url: String, + extra: Map[String, String] = Map.empty, + attributes: Attributes = Attributes()): Artifact = + core.Artifact(url, extra, attributes) + } + + type MavenRepository[G <: core.FetchMetadata] = core.MavenRepository[G] + val MavenRepository: core.MavenRepository.type = core.MavenRepository + } diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 3b36d9694..f8348043f 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -35,7 +35,7 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("ch.qos.logback", "logback-classic"), "1.1.3") val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) - .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here + .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( rootDependencies = Set(dep), @@ -51,7 +51,7 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("org.ow2.asm", "asm-commons"), "5.0.2") val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) - .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here + .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( rootDependencies = Set(dep), @@ -67,7 +67,7 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("joda-time", "joda-time"), "[2.2,2.8]") val res0 = await(resolve(Set(dep), fetchFrom(repositories)).runF) - val res = res0.copy(projectsCache = Map.empty, errors = Map.empty) + val res = res0.copy(projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -75,9 +75,9 @@ object CentralTests extends TestSuite { dep.withCompileScope)) assert(res == expected) - assert(res0.projectsCache.contains(dep.moduleVersion)) + assert(res0.projectCache.contains(dep.moduleVersion)) - val (_, proj) = res0.projectsCache(dep.moduleVersion) + val (_, proj) = res0.projectCache(dep.moduleVersion) assert(proj.version == "2.8") } } diff --git a/core/src/test/scala/coursier/test/ResolutionTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala index d10db84e5..4734bd0f5 100644 --- a/core/src/test/scala/coursier/test/ResolutionTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -168,7 +168,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - errors = Map(dep.moduleVersion -> Seq("Not found")) + errorCache = Map(dep.moduleVersion -> Seq("Not found")) ) assert(res == expected) @@ -185,7 +185,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - projectsCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) + projectCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) ) assert(res == expected) @@ -203,7 +203,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope, trDep.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv, projectsMap(trDep.moduleVersion).kv ) @@ -227,7 +227,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -252,7 +252,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -277,7 +277,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -297,7 +297,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ) @@ -316,7 +316,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -336,7 +336,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -355,7 +355,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -372,7 +372,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -391,7 +391,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -412,7 +412,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -435,7 +435,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -457,7 +457,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -481,7 +481,7 @@ object ResolutionTests extends TestSuite { deps, fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = deps, diff --git a/core/src/test/scala/coursier/test/TestRepository.scala b/core/src/test/scala/coursier/test/TestRepository.scala index b70077b34..fcdf892ff 100644 --- a/core/src/test/scala/coursier/test/TestRepository.scala +++ b/core/src/test/scala/coursier/test/TestRepository.scala @@ -12,8 +12,5 @@ class TestRepository(projects: Map[(Module, String), Project]) extends Repositor EitherT(Task.now( projects.get((module, version)).toRightDisjunction("Not found") )) - def versions(organization: String, name: String, cachePolicy: CachePolicy) = - EitherT(Task.now[String \/ Versions]( - -\/("Not supported") - )) + def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = ??? } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala new file mode 100644 index 000000000..014c56ea9 --- /dev/null +++ b/files/src/main/scala/coursier/Files.scala @@ -0,0 +1,121 @@ +package coursier + +import java.net.URL + +import coursier.core.CachePolicy + +import scala.annotation.tailrec +import scalaz.{-\/, \/-, \/, EitherT} +import scalaz.concurrent.Task + +import java.io._ + +// FIXME This kind of side-effecting API is lame, we should aim at a more functional one. +trait FilesLogger { + def foundLocally(f: File): Unit + def downloadingArtifact(url: String): Unit + def downloadedArtifact(url: String, success: Boolean): Unit +} + +case class Files(cache: Seq[(String, File)], + tmp: () => File, + logger: Option[FilesLogger] = None) { + + def file(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, File] = { + + cache.find{case (base, _) => artifact.url.startsWith(base)} match { + case None => ??? + case Some((base, cacheDir)) => + val file = new File(cacheDir, artifact.url.stripPrefix(base)) + + def locally = { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(file)) + \/-(file) + } + else -\/("Not found in cache") + } + } + + def remote = { + // FIXME A lot of things can go wrong here and are not properly handled: + // - checksums should be validated + // - what if the connection gets closed during the transfer (partial file on disk)? + // - what if someone is trying to write this file at the same time? (no locking of any kind yet) + // - ... + + Task { + try { + file.getParentFile.mkdirs() + + logger.foreach(_.downloadingArtifact(artifact.url)) + + val url = new URL(artifact.url) + val b = Array.fill[Byte](Files.bufferSize)(0) + val in = new BufferedInputStream(url.openStream(), Files.bufferSize) + + try { + val out = new FileOutputStream(file) + try { + @tailrec + def helper(): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + helper() + } + } + + helper() + } finally out.close() + } finally in.close() + + logger.foreach(_.downloadedArtifact(artifact.url, success = true)) + \/-(file) + } + catch { case e: Exception => + logger.foreach(_.downloadedArtifact(artifact.url, success = false)) + -\/(e.getMessage) + } + } + } + + EitherT(cachePolicy(locally)(remote)) + } + } + +} + +object Files { + + var bufferSize = 1024*1024 + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap(_.getMessage) + } + +} diff --git a/project/Coursier.scala b/project/Coursier.scala index 60e22572e..d11e0f0c7 100644 --- a/project/Coursier.scala +++ b/project/Coursier.scala @@ -101,8 +101,20 @@ object CoursierBuild extends Build { ) .enablePlugins(ScalaJSPlugin) - lazy val cli = Project(id = "cli", base = file("cli")) + lazy val files = Project(id = "files", base = file("files")) .dependsOn(coreJvm) + .settings(commonSettings: _*) + .settings( + name := "coursier-files", + libraryDependencies ++= Seq( + "org.http4s" %% "http4s-blazeclient" % "0.8.2", + "com.lihaoyi" %% "utest" % "0.3.0" % "test" + ), + testFrameworks += new TestFramework("utest.runner.Framework") + ) + + lazy val cli = Project(id = "cli", base = file("cli")) + .dependsOn(coreJvm, files) .settings(commonSettings ++ packAutoSettings ++ publishPackTxzArchive ++ publishPackZipArchive: _*) .settings( packArchivePrefix := s"coursier-cli_${scalaBinaryVersion.value}", diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 3cad52b95..528e4cd04 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,7 +1,7 @@ package coursier package web -import coursier.core.{Logger, Remote} +import coursier.core.{DefaultFetchMetadata, Logger} import japgolly.scalajs.react.vdom.{TagMod, Attr} import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope} @@ -18,7 +18,7 @@ case class ResolutionOptions(followOptional: Boolean = false, keepTest: Boolean = false) case class State(modules: Seq[Dependency], - repositories: Seq[Remote], + repositories: Seq[MavenRepository[DefaultFetchMetadata]], options: ResolutionOptions, resolutionOpt: Option[Resolution], editModuleIdx: Int, @@ -71,7 +71,7 @@ class Backend($: BackendScope[Unit, State]) { def updateTree(resolution: Resolution, target: String, reverse: Boolean) = { def depsOf(dep: Dependency) = - resolution.projectsCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter)) + resolution.projectCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter)) val minDependencies = resolution.minDependencies @@ -127,7 +127,7 @@ class Backend($: BackendScope[Unit, State]) { filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) ) - res.last(fetchFrom(s.repositories.map(_.copy(logger = Some(logger)))), 100) + res.last(fetchFrom(s.repositories.map(r => r.copy(fetchMetadata = r.fetchMetadata.copy(logger = Some(logger))))), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context @@ -224,7 +224,7 @@ object App { def depItem(dep: Dependency, finalVersionOpt: Option[String]) = { <.tr( - ^.`class` := (if (res.errors.contains(dep.moduleVersion)) "danger" else ""), + ^.`class` := (if (res.errorCache.contains(dep.moduleVersion)) "danger" else ""), <.td(dep.module.organization), <.td(dep.module.name), <.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")), @@ -234,11 +234,11 @@ object App { if (dep.attributes.classifier.isEmpty) Seq() else Seq(infoLabel(dep.attributes.classifier)), Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq, if (dep.optional) Seq(infoLabel("optional")) else Seq(), - res.errors.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq + res.errorCache.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq )), <.td(Seq[Seq[TagMod]]( - res.projectsCache.get(dep.moduleVersion) match { - case Some((repo: Remote, _)) => + res.projectCache.get(dep.moduleVersion) match { + case Some((MavenRepository(fetchMetadata, _), _)) => // FIXME Maven specific, generalize if/when adding support for Ivy val version0 = finalVersionOpt getOrElse dep.version val relPath = @@ -249,10 +249,10 @@ object App { ) Seq( - <.a(^.href := s"${repo.base}${relPath.mkString("/")}.pom", + <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.pom", <.span(^.`class` := "label label-info", "POM") ), - <.a(^.href := s"${repo.base}${relPath.mkString("/")}.jar", + <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.jar", <.span(^.`class` := "label label-info", "JAR") ) ) @@ -277,7 +277,7 @@ object App { ) ), <.tbody( - sortedDeps.map(dep => depItem(dep, res.projectsCache.get(dep.moduleVersion).map(_._2.version).filter(_ != dep.version))) + sortedDeps.map(dep => depItem(dep, res.projectCache.get(dep.moduleVersion).map(_._2.version).filter(_ != dep.version))) ) ) } @@ -386,19 +386,19 @@ object App { val modules = dependenciesTable("Dependencies") - val repositories = ReactComponentB[Seq[Remote]]("Repositories") + val repositories = ReactComponentB[Seq[MavenRepository[DefaultFetchMetadata]]]("Repositories") .render{ repos => - def repoItem(repo: Remote) = + def repoItem(repo: MavenRepository[DefaultFetchMetadata]) = <.tr( <.td( - <.a(^.href := repo.base, - repo.base + <.a(^.href := repo.fetchMetadata.root, + repo.fetchMetadata.root ) ) ) val sortedRepos = repos - .sortBy(repo => repo.base) + .sortBy(repo => repo.fetchMetadata.root) <.table(^.`class` := "table", <.thead( From c6398592ce8d51c59b8c7097c4ae32dfb350de2e Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:58 +0100 Subject: [PATCH 12/12] Cleaning --- .../src/main/scala/coursier/repository/package.scala | 12 ------------ .../src/main/scala/coursier/repository/package.scala | 0 2 files changed, 12 deletions(-) delete mode 100644 core-jvm/src/main/scala/coursier/repository/package.scala rename {core-js => core}/src/main/scala/coursier/repository/package.scala (100%) diff --git a/core-jvm/src/main/scala/coursier/repository/package.scala b/core-jvm/src/main/scala/coursier/repository/package.scala deleted file mode 100644 index c448f25c5..000000000 --- a/core-jvm/src/main/scala/coursier/repository/package.scala +++ /dev/null @@ -1,12 +0,0 @@ -package coursier - -import coursier.core.DefaultFetchMetadata - -package object repository { - - val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/")) - - val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) - val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) - -} diff --git a/core-js/src/main/scala/coursier/repository/package.scala b/core/src/main/scala/coursier/repository/package.scala similarity index 100% rename from core-js/src/main/scala/coursier/repository/package.scala rename to core/src/main/scala/coursier/repository/package.scala