From d3cd484c15f93be9bf1f3dfbc21ddf4a86a95d6a Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:32 +0100 Subject: [PATCH 01/80] Set version to 1.0.0-SNAPSHOT --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index aab7f9b81..6dc058896 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.1.0-SNAPSHOT" \ No newline at end of file +version in ThisBuild := "1.0.0-SNAPSHOT" From e4dfc862b48e3127350f509936029d128aee05ce Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:32 +0100 Subject: [PATCH 02/80] Replace scopes with Ivy-like configuration --- cli/src/main/scala/coursier/cli/Helper.scala | 2 +- .../scala/coursier/core/Definitions.scala | 22 +-- .../src/main/scala/coursier/core/Orders.scala | 91 +++++---- .../src/main/scala/coursier/core/Parse.scala | 9 - .../main/scala/coursier/core/Resolution.scala | 181 ++++++++++-------- .../coursier/maven/MavenRepository.scala | 8 +- .../src/main/scala/coursier/maven/Pom.scala | 9 +- .../src/main/scala/coursier/package.scala | 8 +- .../scala/coursier/test/CentralTests.scala | 6 +- .../scala/coursier/test/PomParsingTests.scala | 6 +- .../scala/coursier/test/ResolutionTests.scala | 103 +++++----- .../test/scala/coursier/test/package.scala | 16 +- web/src/main/scala/coursier/web/Backend.scala | 27 +-- 13 files changed, 241 insertions(+), 247 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index db82754d4..3f1a98d4d 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -171,7 +171,7 @@ class Helper( } val deps = moduleVersions.map{case (mod, ver) => - Dependency(mod, ver, scope = Scope.Runtime) + Dependency(mod, ver, configuration = "runtime") } val forceVersions = { diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 130b48092..44550acf0 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -24,8 +24,6 @@ case class Module( override def toString = s"$organization:$name" } -sealed abstract class Scope(val name: String) - /** * Dependencies with the same @module will typically see their @version-s merged. * @@ -35,7 +33,7 @@ sealed abstract class Scope(val name: String) case class Dependency( module: Module, version: String, - scope: Scope, + configuration: String, attributes: Attributes, exclusions: Set[(String, String)], optional: Boolean @@ -51,9 +49,10 @@ case class Attributes( case class Project( module: Module, version: String, - dependencies: Seq[Dependency], + dependencies: Seq[(String, Dependency)], parent: Option[(Module, String)], - dependencyManagement: Seq[Dependency], + dependencyManagement: Seq[(String, Dependency)], + configurations: Map[String, Seq[String]], properties: Map[String, String], profiles: Seq[Profile], versions: Option[Versions], @@ -62,23 +61,14 @@ case class Project( def moduleVersion = (module, version) } -object Scope { - case object Compile extends Scope("compile") - case object Runtime extends Scope("runtime") - case object Test extends Scope("test") - case object Provided extends Scope("provided") - case object Import extends Scope("import") - case class Other(override val name: String) extends Scope(name) -} - case class Activation(properties: Seq[(String, Option[String])]) case class Profile( id: String, activeByDefault: Option[Boolean], activation: Activation, - dependencies: Seq[Dependency], - dependencyManagement: Seq[Dependency], + dependencies: Seq[(String, Dependency)], + dependencyManagement: Seq[(String, Dependency)], properties: Map[String, String] ) diff --git a/core/shared/src/main/scala/coursier/core/Orders.scala b/core/shared/src/main/scala/coursier/core/Orders.scala index dd5c7a28a..7df4a87eb 100644 --- a/core/shared/src/main/scala/coursier/core/Orders.scala +++ b/core/shared/src/main/scala/coursier/core/Orders.scala @@ -2,39 +2,56 @@ package coursier.core object Orders { - /** Minimal ad-hoc partial order */ - trait PartialOrder[A] { - /** - * x < y: Some(neg. integer) - * x == y: Some(0) - * x > y: Some(pos. integer) - * x, y not related: None - */ - def cmp(x: A, y: A): Option[Int] + trait PartialOrdering[T] extends scala.math.PartialOrdering[T] { + def lteq(x: T, y: T): Boolean = + tryCompare(x, y) + .exists(_ <= 0) } /** * Only relations: * Compile < Runtime < Test */ - implicit val mavenScopePartialOrder: PartialOrder[Scope] = - new PartialOrder[Scope] { - val higher = Map[Scope, Set[Scope]]( - Scope.Compile -> Set(Scope.Runtime, Scope.Test), - Scope.Runtime -> Set(Scope.Test) - ) + def configurationPartialOrder(configurations: Map[String, Seq[String]]): PartialOrdering[String] = + new PartialOrdering[String] { + def allParents(config: String): Set[String] = { + def helper(configs: Set[String], acc: Set[String]): Set[String] = + if (configs.isEmpty) + acc + else if (configs.exists(acc)) + helper(configs -- acc, acc) + else if (configs.exists(!configurations.contains(_))) { + val (remaining, notFound) = configs.partition(configurations.contains) + helper(remaining, acc ++ notFound) + } else { + val extraConfigs = configs.flatMap(configurations) + helper(extraConfigs, acc ++ configs) + } - def cmp(x: Scope, y: Scope) = - if (x == y) Some(0) - else if (higher.get(x).exists(_(y))) Some(-1) - else if (higher.get(y).exists(_(x))) Some(1) - else None + helper(Set(config), Set.empty) + } + + val allParentsMap = configurations + .keys + .toList + .map(config => config -> (allParents(config) - config)) + .toMap + + def tryCompare(x: String, y: String) = + if (x == y) + Some(0) + else if (allParentsMap.get(x).exists(_(y))) + Some(-1) + else if (allParentsMap.get(y).exists(_(x))) + Some(1) + else + None } /** Non-optional < optional */ - implicit val optionalPartialOrder: PartialOrder[Boolean] = - new PartialOrder[Boolean] { - def cmp(x: Boolean, y: Boolean) = + val optionalPartialOrder: PartialOrdering[Boolean] = + new PartialOrdering[Boolean] { + def tryCompare(x: Boolean, y: Boolean) = Some( if (x == y) 0 else if (x) 1 @@ -51,8 +68,8 @@ object Orders { * * In particular, no exclusions <= anything <= Set(("*", "*")) */ - implicit val exclusionsPartialOrder: PartialOrder[Set[(String, String)]] = - new PartialOrder[Set[(String, String)]] { + val exclusionsPartialOrder: PartialOrdering[Set[(String, String)]] = + new PartialOrdering[Set[(String, String)]] { def boolCmp(a: Boolean, b: Boolean) = (a, b) match { case (true, true) => Some(0) case (true, false) => Some(1) @@ -60,7 +77,7 @@ object Orders { case (false, false) => None } - def cmp(x: Set[(String, String)], y: Set[(String, String)]) = { + def tryCompare(x: Set[(String, String)], y: Set[(String, String)]) = { val (xAll, xExcludeByOrg1, xExcludeByName1, xRemaining0) = Exclusions.partition(x) val (yAll, yExcludeByOrg1, yExcludeByName1, yRemaining0) = Exclusions.partition(y) @@ -102,19 +119,22 @@ object Orders { * Assume all dependencies have same `module`, `version`, and `artifact`; see `minDependencies` * if they don't. */ - def minDependenciesUnsafe(dependencies: Set[Dependency]): Set[Dependency] = { + def minDependenciesUnsafe( + dependencies: Set[Dependency], + configs: ((Module, String)) => Map[String, Seq[String]] + ): Set[Dependency] = { val groupedDependencies = dependencies - .groupBy(dep => (dep.optional, dep.scope)) + .groupBy(dep => (dep.optional, dep.configuration)) .mapValues(deps => deps.head.copy(exclusions = deps.foldLeft(Exclusions.one)((acc, dep) => Exclusions.meet(acc, dep.exclusions)))) .toList val remove = for { List(((xOpt, xScope), xDep), ((yOpt, yScope), yDep)) <- groupedDependencies.combinations(2) - optCmp <- optionalPartialOrder.cmp(xOpt, yOpt).iterator - scopeCmp <- mavenScopePartialOrder.cmp(xScope, yScope).iterator + optCmp <- optionalPartialOrder.tryCompare(xOpt, yOpt).iterator + scopeCmp <- configurationPartialOrder(configs(xDep.moduleVersion)).tryCompare(xScope, yScope).iterator if optCmp*scopeCmp >= 0 - exclCmp <- exclusionsPartialOrder.cmp(xDep.exclusions, yDep.exclusions).iterator + exclCmp <- exclusionsPartialOrder.tryCompare(xDep.exclusions, yDep.exclusions).iterator if optCmp*exclCmp >= 0 if scopeCmp*exclCmp >= 0 xIsMin = optCmp < 0 || scopeCmp < 0 || exclCmp < 0 @@ -130,10 +150,13 @@ object Orders { * * The returned set brings exactly the same things as `dependencies`, with no redundancy. */ - def minDependencies(dependencies: Set[Dependency]): Set[Dependency] = { + def minDependencies( + dependencies: Set[Dependency], + configs: ((Module, String)) => Map[String, Seq[String]] + ): Set[Dependency] = { dependencies - .groupBy(_.copy(scope = Scope.Other(""), exclusions = Set.empty, optional = false)) - .mapValues(minDependenciesUnsafe) + .groupBy(_.copy(configuration = "", exclusions = Set.empty, optional = false)) + .mapValues(minDependenciesUnsafe(_, configs)) .valuesIterator .fold(Set.empty)(_ ++ _) } diff --git a/core/shared/src/main/scala/coursier/core/Parse.scala b/core/shared/src/main/scala/coursier/core/Parse.scala index 4d89c35ea..3259d1ef1 100644 --- a/core/shared/src/main/scala/coursier/core/Parse.scala +++ b/core/shared/src/main/scala/coursier/core/Parse.scala @@ -4,15 +4,6 @@ import coursier.core.compatibility._ object Parse { - def scope(s: String): Scope = s match { - case "compile" => Scope.Compile - case "runtime" => Scope.Runtime - case "test" => Scope.Test - case "provided" => Scope.Provided - case "import" => Scope.Import - case other => Scope.Other(other) - } - def version(s: String): Option[Version] = { if (s.isEmpty || s.exists(c => c != '.' && c != '-' && c != '_' && !c.letterOrDigit)) None else Some(Version(s)) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index fa4203bdd..891afee00 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -37,22 +37,22 @@ object Resolution { (dep.module.organization, dep.module.name, dep.attributes.`type`) def add( - dict: Map[Key, Dependency], - dep: Dependency - ): Map[Key, Dependency] = { + dict: Map[Key, (String, Dependency)], + item: (String, Dependency) + ): Map[Key, (String, Dependency)] = { - val key0 = key(dep) + val key0 = key(item._2) if (dict.contains(key0)) dict else - dict + (key0 -> dep) + dict + (key0 -> item) } def addSeq( - dict: Map[Key, Dependency], - deps: Seq[Dependency] - ): Map[Key, Dependency] = + dict: Map[Key, (String, Dependency)], + deps: Seq[(String, Dependency)] + ): Map[Key, (String, Dependency)] = (dict /: deps)(add) } @@ -62,14 +62,14 @@ object Resolution { ): Map[String, String] = dict ++ other.filterKeys(!dict.contains(_)) - def addDependencies(deps: Seq[Seq[Dependency]]): Seq[Dependency] = { + def addDependencies(deps: Seq[Seq[(String, Dependency)]]): Seq[(String, Dependency)] = { val res = - (deps :\ (Set.empty[DepMgmt.Key], Seq.empty[Dependency])) { + (deps :\ (Set.empty[DepMgmt.Key], Seq.empty[(String, Dependency)])) { case (deps0, (set, acc)) => val deps = deps0 - .filter(dep => !set(DepMgmt.key(dep))) + .filter{case (_, dep) => !set(DepMgmt.key(dep))} - (set ++ deps.map(DepMgmt.key), acc ++ deps) + (set ++ deps.map{case (_, dep) => DepMgmt.key(dep)}, acc ++ deps) } res._2 @@ -83,9 +83,9 @@ object Resolution { * Substitutes `properties` in `dependencies`. */ def withProperties( - dependencies: Seq[Dependency], + dependencies: Seq[(String, Dependency)], properties: Map[String, String] - ): Seq[Dependency] = { + ): Seq[(String, Dependency)] = { def substituteProps(s: String) = { val matches = propRegex @@ -107,8 +107,8 @@ object Resolution { } dependencies - .map{ dep => - dep.copy( + .map {case (config, dep) => + substituteProps(config) -> dep.copy( module = dep.module.copy( organization = substituteProps(dep.module.organization), name = substituteProps(dep.module.name) @@ -118,7 +118,7 @@ object Resolution { `type` = substituteProps(dep.attributes.`type`), classifier = substituteProps(dep.attributes.classifier) ), - scope = Parse.scope(substituteProps(dep.scope.name)), + configuration = substituteProps(dep.configuration), exclusions = dep.exclusions .map{case (org, name) => (substituteProps(org), substituteProps(name)) @@ -209,25 +209,6 @@ object Resolution { ) } - /** - * If one of our dependency has scope `base`, and a transitive - * dependency of it has scope `transitive`, return the scope of - * the latter for us, if any. If empty, means the transitive dependency - * should not be considered a dependency for us. - * - * See https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Scope. - */ - def resolveScope( - base: Scope, - transitive: Scope - ): Option[Scope] = - (base, transitive) match { - case (other, Scope.Compile) => Some(other) - case (Scope.Compile, Scope.Runtime) => Some(Scope.Runtime) - case (other, Scope.Runtime) => Some(other) - case _ => None - } - /** * Applies `dependencyManagement` to `dependencies`. * @@ -235,36 +216,39 @@ object Resolution { * `dependencyManagement`. */ def depsWithDependencyManagement( - dependencies: Seq[Dependency], - dependencyManagement: Seq[Dependency] - ): Seq[Dependency] = { + dependencies: Seq[(String, Dependency)], + dependencyManagement: Seq[(String, Dependency)] + ): Seq[(String, Dependency)] = { // See http://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Management lazy val dict = DepMgmt.addSeq(Map.empty, dependencyManagement) dependencies - .map { dep0 => + .map {case (config0, dep0) => + var config = config0 var dep = dep0 - for (mgmtDep <- dict.get(DepMgmt.key(dep0))) { + for ((mgmtConfig, mgmtDep) <- dict.get(DepMgmt.key(dep0))) { if (dep.version.isEmpty) dep = dep.copy(version = mgmtDep.version) - if (dep.scope.name.isEmpty) - dep = dep.copy(scope = mgmtDep.scope) + if (config.isEmpty) + config = mgmtConfig if (dep.exclusions.isEmpty) dep = dep.copy(exclusions = mgmtDep.exclusions) } - dep + (config, dep) } } - def withDefaultScope(dep: Dependency): Dependency = - if (dep.scope.name.isEmpty) - dep.copy(scope = Scope.Compile) + val defaultConfiguration = "compile" + + def withDefaultConfig(dep: Dependency): Dependency = + if (dep.configuration.isEmpty) + dep.copy(configuration = defaultConfiguration) else dep @@ -272,19 +256,37 @@ object Resolution { * Filters `dependencies` with `exclusions`. */ def withExclusions( - dependencies: Seq[Dependency], + dependencies: Seq[(String, Dependency)], exclusions: Set[(String, String)] - ): Seq[Dependency] = { + ): Seq[(String, Dependency)] = { val filter = Exclusions(exclusions) dependencies - .filter(dep => filter(dep.module.organization, dep.module.name)) - .map(dep => - dep.copy( + .filter{case (_, dep) => filter(dep.module.organization, dep.module.name) } + .map{case (config, dep) => + config -> dep.copy( exclusions = Exclusions.minimize(dep.exclusions ++ exclusions) ) - ) + } + } + + def withParentConfigurations(config: String, configurations: Map[String, Seq[String]]): Set[String] = { + @tailrec + def helper(configs: Set[String], acc: Set[String]): Set[String] = + if (configs.isEmpty) + acc + else if (configs.exists(acc)) + helper(configs -- acc, acc) + else if (configs.exists(!configurations.contains(_))) { + val (remaining, notFound) = configs.partition(configurations.contains) + helper(remaining, acc ++ notFound) + } else { + val extraConfigs = configs.flatMap(configurations) + helper(extraConfigs, acc ++ configs) + } + + helper(Set(config), Set.empty) } /** @@ -312,29 +314,33 @@ object Resolution { ) ) - val deps = - withExclusions( - depsWithDependencyManagement( - // Important: properties have to be applied to both, - // so that dep mgmt can be matched properly - // Tested with org.ow2.asm:asm-commons:5.0.2 in CentralTests - withProperties(project.dependencies, properties), - withProperties(project.dependencyManagement, properties) - ), - from.exclusions - ) - .map(withDefaultScope) + val configurations = withParentConfigurations(from.configuration, project.configurations) - deps - .flatMap { trDep => - resolveScope(from.scope, trDep.scope) - .map(scope => - trDep.copy( - scope = scope, - optional = trDep.optional || from.optional - ) - ) - } + withExclusions( + depsWithDependencyManagement( + // Important: properties have to be applied to both, + // so that dep mgmt can be matched properly + // Tested with org.ow2.asm:asm-commons:5.0.2 in CentralTests + withProperties(project.dependencies, properties), + withProperties(project.dependencyManagement, properties) + ), + from.exclusions + ) + .map{ + case (config, dep) => + (if (config.isEmpty) defaultConfiguration else config) -> { + if (dep.configuration.isEmpty) + dep.copy(configuration = defaultConfiguration) + else + dep + } + } + .collect{case (config, dep) if configurations(config) => + if (from.optional) + dep.copy(optional = true) + else + dep + } } /** @@ -438,7 +444,7 @@ case class Resolution( def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = // TODO Provide the modules whose version was forced by dependency overrides too merge( - rootDependencies.map(withDefaultScope) ++ dependencies ++ transitiveDependencies, + rootDependencies.map(withDefaultConfig) ++ dependencies ++ transitiveDependencies, forceVersions ) @@ -507,7 +513,7 @@ case class Resolution( */ def remainingDependencies: Set[Dependency] = { val rootDependencies0 = rootDependencies - .map(withDefaultScope) + .map(withDefaultConfig) .map(eraseVersion) @tailrec @@ -599,7 +605,7 @@ case class Resolution( val modules = (project.dependencies ++ profileDependencies) .collect{ - case dep if dep.scope == Scope.Import => dep.moduleVersion + case ("import", dep) => dep.moduleVersion } modules.toSet ++ project.parent @@ -679,7 +685,7 @@ case class Resolution( val deps = ( dependencies0 - .collect { case dep if dep.scope == Scope.Import => + .collect { case ("import", dep) => dep.moduleVersion } ++ project.parent @@ -693,16 +699,16 @@ case class Resolution( profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement) ) - ).foldLeft(Map.empty[DepMgmt.Key, Dependency])(DepMgmt.addSeq) + ).foldLeft(Map.empty[DepMgmt.Key, (String, Dependency)])(DepMgmt.addSeq) val depsSet = deps.toSet project.copy( dependencies = dependencies0 - .filterNot(dep => - dep.scope == Scope.Import && depsSet(dep.moduleVersion) - ) ++ + .filterNot{case (config, dep) => + config == "import" && depsSet(dep.moduleVersion) + } ++ project.parent .filter(projectCache.contains) .toSeq @@ -716,7 +722,14 @@ case class Resolution( } def minDependencies: Set[Dependency] = - Orders.minDependencies(dependencies) + Orders.minDependencies( + dependencies, + dep => + projectCache + .get(dep) + .map(_._2.configurations) + .getOrElse(Map.empty) + ) def artifacts: Seq[Artifact] = for { diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 3ea96fc81..b2b13f3fe 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -35,6 +35,12 @@ object MavenRepository { .map(_.value) .filter(_.nonEmpty) + + val defaultConfigurations = Map( + "runtime" -> Seq("compile"), + "test" -> Seq("runtime") + ) + } case class MavenRepository( @@ -231,7 +237,7 @@ case class MavenRepository( xml <- \/.fromEither(compatibility.xmlParse(str)) _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") proj <- Pom.project(xml) - } yield proj): (String \/ Project) + } yield proj.copy(configurations = defaultConfigurations)): (String \/ Project) } } } diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 8033498fa..4cd412d2a 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -43,16 +43,14 @@ object Pom { private def readVersion(node: Node) = text(node, "version", "Version").getOrElse("").trim - private val defaultScope = Scope.Other("") private val defaultType = "jar" private val defaultClassifier = "" - def dependency(node: Node): String \/ Dependency = { + def dependency(node: Node): String \/ (String, Dependency) = { for { mod <- module(node) version0 = readVersion(node) scopeOpt = text(node, "scope", "").toOption - .map(Parse.scope) typeOpt = text(node, "type", "").toOption classifierOpt = text(node, "classifier", "").toOption xmlExclusions = node.child @@ -64,10 +62,10 @@ object Pom { xmlExclusions.toList.traverseU(module(_)) } optional = text(node, "optional", "").toOption.toSeq.contains("true") - } yield Dependency( + } yield scopeOpt.getOrElse("") -> Dependency( mod, version0, - scopeOpt getOrElse defaultScope, + "", Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), exclusions.map(mod => (mod.organization, mod.name)).toSet, optional @@ -191,6 +189,7 @@ object Pom { deps, parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), depMgmts, + Map.empty, properties.toMap, profiles, None, diff --git a/core/shared/src/main/scala/coursier/package.scala b/core/shared/src/main/scala/coursier/package.scala index 6803f43ed..c815d5dcd 100644 --- a/core/shared/src/main/scala/coursier/package.scala +++ b/core/shared/src/main/scala/coursier/package.scala @@ -9,8 +9,8 @@ package object coursier { def apply( module: Module, version: String, - // Substituted by Resolver with its own default scope (compile) - scope: Scope = Scope.Other(""), + // Substituted by Resolver with its own default configuration (compile) + configuration: String = "", attributes: Attributes = Attributes(), exclusions: Set[(String, String)] = Set.empty, optional: Boolean = false @@ -18,7 +18,7 @@ package object coursier { core.Dependency( module, version, - scope, + configuration, attributes, exclusions, optional @@ -48,8 +48,6 @@ package object coursier { type ModuleVersion = (core.Module, String) - type Scope = core.Scope - val Scope = core.Scope type Repository = core.Repository val Repository = core.Repository diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index 751397d13..7ea36ea21 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -44,7 +44,8 @@ object CentralTests extends TestSuite { def resolutionCheck( module: Module, version: String, - extraRepo: Option[Repository] = None + extraRepo: Option[Repository] = None, + configuration: String = "" ) = async { val expected = @@ -54,7 +55,7 @@ object CentralTests extends TestSuite { .split('\n') .toSeq - val dep = Dependency(module, version) + val dep = Dependency(module, version, configuration = configuration) val res = await(resolve(Set(dep), extraRepo = extraRepo)) val result = res @@ -138,6 +139,7 @@ object CentralTests extends TestSuite { resolutionCheck( Module("com.github.fommil", "java-logging"), "1.2-SNAPSHOT", + configuration = "runtime", extraRepo = Some(MavenRepository("https://oss.sonatype.org/content/repositories/public/")) ) } diff --git a/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala index 6bea7b696..833cabbbb 100644 --- a/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala +++ b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala @@ -21,7 +21,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra"))) + val expected = \/-("" -> Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra"))) val result = Pom.dependency(xmlParse(depNode).right.get) @@ -90,7 +90,7 @@ object PomParsingTests extends TestSuite { None, Profile.Activation(Nil), Seq( - Dependency(Module("comp", "lib"), "0.2")), + "" -> Dependency(Module("comp", "lib"), "0.2")), Nil, Map.empty )) @@ -122,7 +122,7 @@ object PomParsingTests extends TestSuite { Profile.Activation(Nil), Nil, Seq( - Dependency(Module("comp", "lib"), "0.2", scope = Scope.Test)), + "test" -> Dependency(Module("comp", "lib"), "0.2")), Map.empty )) diff --git a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala index fd9d0ea4d..627ed288b 100644 --- a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala +++ b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala @@ -2,6 +2,7 @@ package coursier package test import coursier.core.Repository +import coursier.maven.MavenRepository import utest._ import scala.async.Async.{ async, await } @@ -27,69 +28,68 @@ object ResolutionTests extends TestSuite { Project(Module("acme", "config"), "1.3.0"), Project(Module("acme", "play"), "2.4.0", Seq( - Dependency(Module("acme", "play-json"), "2.4.0"))), + "" -> Dependency(Module("acme", "play-json"), "2.4.0"))), Project(Module("acme", "play-json"), "2.4.0"), Project(Module("acme", "play"), "2.4.1", dependencies = Seq( - Dependency(Module("acme", "play-json"), "${playJsonVersion}"), - Dependency(Module("${project.groupId}", "${configName}"), "1.3.0")), + "" -> Dependency(Module("acme", "play-json"), "${playJsonVersion}"), + "" -> Dependency(Module("${project.groupId}", "${configName}"), "1.3.0")), properties = Map( "playJsonVersion" -> "2.4.0", "configName" -> "config")), Project(Module("acme", "play-extra-no-config"), "2.4.1", Seq( - Dependency(Module("acme", "play"), "2.4.1", + "" -> Dependency(Module("acme", "play"), "2.4.1", exclusions = Set(("acme", "config"))))), Project(Module("acme", "play-extra-no-config-no"), "2.4.1", Seq( - Dependency(Module("acme", "play"), "2.4.1", + "" -> Dependency(Module("acme", "play"), "2.4.1", exclusions = Set(("*", "config"))))), Project(Module("hudsucker", "mail"), "10.0", Seq( - Dependency(Module("${project.groupId}", "test-util"), "${project.version}", - scope = Scope.Test))), + "test" -> Dependency(Module("${project.groupId}", "test-util"), "${project.version}"))), Project(Module("hudsucker", "test-util"), "10.0"), Project(Module("se.ikea", "parent"), "18.0", dependencyManagement = Seq( - Dependency(Module("acme", "play"), "2.4.0", + "" -> Dependency(Module("acme", "play"), "2.4.0", exclusions = Set(("acme", "play-json"))))), Project(Module("se.ikea", "billy"), "18.0", dependencies = Seq( - Dependency(Module("acme", "play"), "")), + "" -> Dependency(Module("acme", "play"), "")), parent = Some(Module("se.ikea", "parent"), "18.0")), Project(Module("org.gnome", "parent"), "7.0", Seq( - Dependency(Module("org.gnu", "glib"), "13.4"))), + "" -> Dependency(Module("org.gnu", "glib"), "13.4"))), Project(Module("org.gnome", "panel-legacy"), "7.0", dependencies = Seq( - Dependency(Module("org.gnome", "desktop"), "${project.version}")), + "" -> Dependency(Module("org.gnome", "desktop"), "${project.version}")), parent = Some(Module("org.gnome", "parent"), "7.0")), Project(Module("gov.nsa", "secure-pgp"), "10.0", Seq( - Dependency(Module("gov.nsa", "crypto"), "536.89"))), + "" -> Dependency(Module("gov.nsa", "crypto"), "536.89"))), Project(Module("com.mailapp", "mail-client"), "2.1", dependencies = Seq( - Dependency(Module("gov.nsa", "secure-pgp"), "10.0", + "" -> Dependency(Module("gov.nsa", "secure-pgp"), "10.0", exclusions = Set(("*", "${crypto.name}")))), properties = Map("crypto.name" -> "crypto", "dummy" -> "2")), Project(Module("com.thoughtworks.paranamer", "paranamer-parent"), "2.6", dependencies = Seq( - Dependency(Module("junit", "junit"), "")), + "" -> Dependency(Module("junit", "junit"), "")), dependencyManagement = Seq( - Dependency(Module("junit", "junit"), "4.11", scope = Scope.Test))), + "test" -> Dependency(Module("junit", "junit"), "4.11"))), Project(Module("com.thoughtworks.paranamer", "paranamer"), "2.6", parent = Some(Module("com.thoughtworks.paranamer", "paranamer-parent"), "2.6")), @@ -97,33 +97,33 @@ object ResolutionTests extends TestSuite { Project(Module("com.github.dummy", "libb"), "0.3.3", profiles = Seq( Profile("default", activeByDefault = Some(true), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.4.2", dependencies = Seq( - Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4")), + "" -> Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4")), profiles = Seq( Profile("default", activeByDefault = Some(true), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"), - Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4", scope = Scope.Test))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"), + "test" -> Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4"))))), Project(Module("com.github.dummy", "libb"), "0.5.3", properties = Map("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> None)), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.5.4", properties = Map("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("true"))), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.5.5", properties = Map("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb-parent"), "0.5.6", properties = Map("special" -> "true")), @@ -133,39 +133,39 @@ object ResolutionTests extends TestSuite { properties = Map("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("an-org", "a-name"), "1.0"), Project(Module("an-org", "a-name"), "1.2"), Project(Module("an-org", "a-lib"), "1.0", - Seq(Dependency(Module("an-org", "a-name"), "1.0"))), + Seq("" -> Dependency(Module("an-org", "a-name"), "1.0"))), Project(Module("an-org", "a-lib"), "1.1"), Project(Module("an-org", "a-lib"), "1.2", - Seq(Dependency(Module("an-org", "a-name"), "1.2"))), + Seq("" -> Dependency(Module("an-org", "a-name"), "1.2"))), Project(Module("an-org", "another-lib"), "1.0", - Seq(Dependency(Module("an-org", "a-name"), "1.0"))), + Seq("" -> Dependency(Module("an-org", "a-name"), "1.0"))), // Must bring transitively an-org:a-name, as an optional dependency Project(Module("an-org", "an-app"), "1.0", Seq( - Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))), - Dependency(Module("an-org", "another-lib"), "1.0", optional = true))), + "" -> Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))), + "" -> Dependency(Module("an-org", "another-lib"), "1.0", optional = true))), Project(Module("an-org", "an-app"), "1.1", Seq( - Dependency(Module("an-org", "a-lib"), "1.1"))), + "" -> Dependency(Module("an-org", "a-lib"), "1.1"))), Project(Module("an-org", "an-app"), "1.2", Seq( - Dependency(Module("an-org", "a-lib"), "1.2"))) + "" -> Dependency(Module("an-org", "a-lib"), "1.2"))) ) - val projectsMap = projects.map(p => p.moduleVersion -> p).toMap + val projectsMap = projects.map(p => p.moduleVersion -> p.copy(configurations = MavenRepository.defaultConfigurations)).toMap val testRepository = new TestRepository(projectsMap) val repositories = Seq[Repository]( @@ -308,8 +308,7 @@ object ResolutionTests extends TestSuite { async { val dep = Dependency(Module("hudsucker", "mail"), "10.0") val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None) val expected = Resolution( @@ -331,8 +330,7 @@ object ResolutionTests extends TestSuite { exclusions = Set(("acme", "play-json"))) ) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( @@ -350,8 +348,7 @@ object ResolutionTests extends TestSuite { Dependency(Module("org.gnu", "glib"), "13.4"), Dependency(Module("org.gnome", "desktop"), "7.0")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -368,8 +365,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("gov.nsa", "secure-pgp"), "10.0", exclusions = Set(("*", "crypto")))) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -384,8 +380,7 @@ object ResolutionTests extends TestSuite { async { val dep = Dependency(Module("com.thoughtworks.paranamer", "paranamer"), "2.6") val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -402,8 +397,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -422,8 +416,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -444,8 +437,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -466,7 +458,7 @@ object ResolutionTests extends TestSuite { Dependency(Module("an-org", "a-name"), "1.0", optional = true)) val res = await(resolve0( Set(dep), - filter = Some(_.scope == Scope.Compile) + filter = Some(_ => true) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -489,7 +481,7 @@ object ResolutionTests extends TestSuite { Dependency(Module("an-org", "a-name"), "1.0", optional = true)) val res = await(resolve0( deps, - filter = Some(_.scope == Scope.Compile) + filter = Some(_ => true) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -511,8 +503,7 @@ object ResolutionTests extends TestSuite { val res = await(resolve0( deps, - forceVersions = depOverrides, - filter = Some(_.scope == Scope.Compile) + forceVersions = depOverrides )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -536,8 +527,7 @@ object ResolutionTests extends TestSuite { val res = await(resolve0( deps, - forceVersions = depOverrides, - filter = Some(_.scope == Scope.Compile) + forceVersions = depOverrides )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -563,8 +553,7 @@ object ResolutionTests extends TestSuite { val res = await(resolve0( deps, - forceVersions = depOverrides, - filter = Some(_.scope == Scope.Compile) + forceVersions = depOverrides )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -585,9 +574,9 @@ object ResolutionTests extends TestSuite { 'propertySubstitution{ val res = core.Resolution.withProperties( - Seq(Dependency(Module("a-company", "a-name"), "${a.property}")), + Seq("" -> Dependency(Module("a-company", "a-name"), "${a.property}")), Map("a.property" -> "a-version")) - val expected = Seq(Dependency(Module("a-company", "a-name"), "a-version")) + val expected = Seq("" -> Dependency(Module("a-company", "a-name"), "a-version")) assert(res == expected) } diff --git a/tests/shared/src/test/scala/coursier/test/package.scala b/tests/shared/src/test/scala/coursier/test/package.scala index ebcf0b6ab..dc8d31315 100644 --- a/tests/shared/src/test/scala/coursier/test/package.scala +++ b/tests/shared/src/test/scala/coursier/test/package.scala @@ -3,7 +3,7 @@ package coursier package object test { implicit class DependencyOps(val underlying: Dependency) extends AnyVal { - def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile) + def withCompileScope: Dependency = underlying.copy(configuration = "compile") } object Profile { @@ -16,8 +16,8 @@ package object test { def apply(id: String, activeByDefault: Option[Boolean] = None, activation: Activation = Activation(), - dependencies: Seq[Dependency] = Nil, - dependencyManagement: Seq[Dependency] = Nil, + dependencies: Seq[(String, Dependency)] = Nil, + dependencyManagement: Seq[(String, Dependency)] = Nil, properties: Map[String, String] = Map.empty) = core.Profile(id, activeByDefault, activation, dependencies, dependencyManagement, properties) } @@ -25,13 +25,15 @@ package object test { object Project { def apply(module: Module, version: String, - dependencies: Seq[Dependency] = Seq.empty, + dependencies: Seq[(String, Dependency)] = Seq.empty, parent: Option[ModuleVersion] = None, - dependencyManagement: Seq[Dependency] = Seq.empty, + dependencyManagement: Seq[(String, Dependency)] = Seq.empty, + configurations: Map[String, Seq[String]] = Map.empty, properties: Map[String, String] = Map.empty, profiles: Seq[Profile] = Seq.empty, versions: Option[core.Versions] = None, - snapshotVersioning: Option[core.SnapshotVersioning] = None): Project = - core.Project(module, version, dependencies, parent, dependencyManagement, properties, profiles, versions, snapshotVersioning) + snapshotVersioning: Option[core.SnapshotVersioning] = None + ): Project = + core.Project(module, version, dependencies, parent, dependencyManagement, configurations, properties, profiles, versions, snapshotVersioning) } } diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 56117eb1c..8ae7d05a9 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -17,8 +17,7 @@ import scala.scalajs.js import js.Dynamic.{ global => g } case class ResolutionOptions( - followOptional: Boolean = false, - keepTest: Boolean = false + followOptional: Boolean = false ) case class State( @@ -66,7 +65,7 @@ class Backend($: BackendScope[Unit, State]) { Seq( dep.module.organization, dep.module.name, - dep.scope.name + dep.configuration ).mkString(":") for { @@ -176,8 +175,7 @@ class Backend($: BackendScope[Unit, State]) { val res = coursier.Resolution( s.modules.toSet, filter = Some(dep => - (s.options.followOptional || !dep.optional) && - (s.options.keepTest || dep.scope != Scope.Test) + s.options.followOptional || !dep.optional ) ) @@ -338,14 +336,6 @@ class Backend($: BackendScope[Unit, State]) { ) ) } - def toggleTest(e: ReactEventI) = { - $.modState(s => - s.copy( - options = s.options - .copy(keepTest = !s.options.keepTest) - ) - ) - } } } @@ -380,7 +370,7 @@ object App { <.td(dep.module.name), <.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")), <.td(Seq[Seq[TagMod]]( - if (dep.scope == Scope.Compile) Seq() else Seq(infoLabel(dep.scope.name)), + if (dep.configuration == "compile") Seq() else Seq(infoLabel(dep.configuration)), if (dep.attributes.`type`.isEmpty || dep.attributes.`type` == "jar") Seq() else Seq(infoLabel(dep.attributes.`type`)), if (dep.attributes.classifier.isEmpty) Seq() else Seq(infoLabel(dep.attributes.classifier)), Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq, @@ -683,15 +673,6 @@ object App { "Follow optional dependencies" ) ) - ), - <.div(^.`class` := "checkbox", - <.label( - <.input(^.`type` := "checkbox", - ^.onChange ==> backend.options.toggleTest, - if (options.keepTest) Seq(^.checked := "checked") else Seq(), - "Keep test dependencies" - ) - ) ) ) } From 3b4b773c6433de070d19239b1ad182037e5b413c Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:32 +0100 Subject: [PATCH 03/80] Add Ivy repository support --- .../main/scala/coursier/cli/Coursier.scala | 2 + cli/src/main/scala/coursier/cli/Helper.scala | 107 +++++---- .../coursier/core/compatibility/package.scala | 4 +- .../coursier/core/compatibility/package.scala | 17 +- .../scala/coursier/core/Definitions.scala | 29 ++- .../src/main/scala/coursier/core/Orders.scala | 16 +- .../src/main/scala/coursier/core/Parse.scala | 17 ++ .../main/scala/coursier/core/Resolution.scala | 23 +- .../scala/coursier/ivy/IvyRepository.scala | 214 ++++++++++++++++++ .../src/main/scala/coursier/ivy/IvyXml.scala | 127 +++++++++++ .../coursier/maven/MavenRepository.scala | 3 +- .../src/main/scala/coursier/maven/Pom.scala | 82 +++---- .../src/main/scala/coursier/package.scala | 2 +- .../src/main/scala/coursier/util/Xml.scala | 30 ++- .../scala/coursier/test/PomParsingTests.scala | 2 +- .../test/scala/coursier/test/package.scala | 61 +++-- 16 files changed, 601 insertions(+), 135 deletions(-) create mode 100644 core/shared/src/main/scala/coursier/ivy/IvyRepository.scala create mode 100644 core/shared/src/main/scala/coursier/ivy/IvyXml.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 9dd707773..bc7d2eb8f 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -30,6 +30,8 @@ case class CommonOptions( @HelpMessage("Repositories - for multiple repositories, separate with comma and/or repeat this option (e.g. -r central,ivy2local -r sonatype-snapshots, or equivalently -r central,ivy2local,sonatype-snapshots)") @ExtraName("r") repository: List[String], + @HelpMessage("Do not add default repositories (~/.ivy2/local, and Central)") + noDefault: Boolean = false, @HelpMessage("Force module version") @ValueDescription("organization:name:forcedVersion") @ExtraName("V") diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 3f1a98d4d..459d32f84 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -4,6 +4,8 @@ package cli import java.io.{ OutputStreamWriter, File } import java.util.UUID +import coursier.ivy.IvyRepository + import scalaz.{ \/-, -\/ } import scalaz.concurrent.Task @@ -62,69 +64,66 @@ class Helper( else CachePolicy.Default - val cache = Cache(new File(cacheOptions.cache)) - cache.init(verbose = verbose0 >= 0) + val files = + Files( + Seq( + "http://" -> new File(new File(cacheOptions.cache), "http"), + "https://" -> new File(new File(cacheOptions.cache), "https") + ), + () => ???, + concurrentDownloadCount = parallel + ) - val repositoryIds = { - val repositoryIds0 = repository - .flatMap(_.split(',')) - .map(_.trim) - .filter(_.nonEmpty) + val central = MavenRepository("https://repo1.maven.org/maven2/") + val ivy2Local = MavenRepository( + new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, + ivyLike = true + ) + val defaultRepositories = Seq( + ivy2Local, + central + ) - if (repositoryIds0.isEmpty) - cache.default() - else - repositoryIds0 - } + val repositories0 = common.repository.map { repo => + val repo0 = repo.toLowerCase + if (repo0 == "central") + Right(central) + else if (repo0 == "ivy2local") + Right(ivy2Local) + else if (repo0.startsWith("sonatype:")) + Right( + MavenRepository(s"https://oss.sonatype.org/content/repositories/${repo.drop("sonatype:".length)}") + ) + else { + val (url, r) = + if (repo.startsWith("ivy:")) { + val url = repo.drop("ivy:".length) + (url, IvyRepository(url)) + } else if (repo.startsWith("ivy-like:")) { + val url = repo.drop("ivy-like:".length) + (url, MavenRepository(url, ivyLike = true)) + } else { + (repo, MavenRepository(repo)) + } - val repoMap = cache.map() - val repoByBase = repoMap.map { case (_, v @ (m, _)) => - m.root -> v - } - - val repositoryIdsOpt0 = repositoryIds.map { id => - repoMap.get(id) match { - case Some(v) => Right(v) - case None => - if (id.contains("://")) { - val root0 = if (id.endsWith("/")) id else id + "/" - Right( - repoByBase.getOrElse(root0, { - val id0 = UUID.randomUUID().toString - if (verbose0 >= 1) - Console.err.println(s"Addding repository $id0 ($root0)") - - // FIXME This could be done more cleanly - cache.add(id0, root0, ivyLike = false) - cache.map().getOrElse(id0, - sys.error(s"Adding repository $id0 ($root0)") - ) - }) - ) - } else - Left(id) + if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/")) + Right(r) + else + Left(repo -> s"Unrecognized protocol or repository: $url") } } - val notFoundRepositoryIds = repositoryIdsOpt0.collect { - case Left(id) => id - } - - if (notFoundRepositoryIds.nonEmpty) { - errPrintln( - (if (notFoundRepositoryIds.lengthCompare(1) == 0) "Repository" else "Repositories") + - " not found: " + - notFoundRepositoryIds.mkString(", ") - ) - + val unrecognizedRepos = repositories0.collect { case Left(e) => e } + if (unrecognizedRepos.nonEmpty) { + errPrintln(s"${unrecognizedRepos.length} error(s) parsing repositories:") + for ((repo, err) <- unrecognizedRepos) + errPrintln(s"$repo: $err") sys.exit(255) } - val files = cache.files().copy(concurrentDownloadCount = parallel) - - val (repositories, fileCaches) = repositoryIdsOpt0 - .collect { case Right(v) => v } - .unzip + val repositories = + (if (common.noDefault) Nil else defaultRepositories) ++ + repositories0.collect { case Right(r) => r } val (rawDependencies, extraArgs) = { val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) diff --git a/core/js/src/main/scala/coursier/core/compatibility/package.scala b/core/js/src/main/scala/coursier/core/compatibility/package.scala index 9226f6d07..f7659c987 100644 --- a/core/js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/js/src/main/scala/coursier/core/compatibility/package.scala @@ -46,11 +46,13 @@ package object compatibility { def label = option[String](node0.nodeName) .getOrElse("") - def child = + def children = option[NodeList](node0.childNodes) .map(l => List.tabulate(l.length)(l.item).map(fromNode)) .getOrElse(Nil) + def attributes: Seq[(String, String)] = ??? + // `exists` instead of `contains`, for scala 2.10 def isText = option[Int](node0.nodeType) diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index 153b376e0..b4874ba94 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -2,6 +2,8 @@ package coursier.core import coursier.util.Xml +import scala.xml.{ MetaData, Null } + package object compatibility { implicit class RichChar(val c: Char) extends AnyVal { @@ -16,8 +18,21 @@ package object compatibility { def fromNode(node: scala.xml.Node): Xml.Node = new Xml.Node { + lazy val attributes = { + def helper(m: MetaData): Stream[(String, String)] = + m match { + case Null => Stream.empty + case attr => + val value = attr.value.collect { + case scala.xml.Text(t) => t + }.mkString("") + (attr.key -> value) #:: helper(m.next) + } + + helper(node.attributes).toVector + } def label = node.label - def child = node.child.map(fromNode) + def children = node.child.map(fromNode) def isText = node match { case _: scala.xml.Text => true; case _ => false } def textContent = node.text def isElement = node match { case _: scala.xml.Elem => true; case _ => false } diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 44550acf0..e9b52a889 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -34,13 +34,16 @@ case class Dependency( module: Module, version: String, configuration: String, - attributes: Attributes, exclusions: Set[(String, String)], + + // Maven-specific + attributes: Attributes, optional: Boolean ) { def moduleVersion = (module, version) } +// Maven-specific case class Attributes( `type`: String, classifier: String @@ -49,20 +52,30 @@ case class Attributes( case class Project( module: Module, version: String, + // First String is configuration (scope for Maven) dependencies: Seq[(String, Dependency)], + // For Maven, this is the standard scopes as an Ivy configuration + configurations: Map[String, Seq[String]], + + // Maven-specific parent: Option[(Module, String)], dependencyManagement: Seq[(String, Dependency)], - configurations: Map[String, Seq[String]], properties: Map[String, String], profiles: Seq[Profile], versions: Option[Versions], - snapshotVersioning: Option[SnapshotVersioning] + snapshotVersioning: Option[SnapshotVersioning], + + // Ivy-specific + // First String is configuration + publications: Seq[(String, Publication)] ) { def moduleVersion = (module, version) } +// Maven-specific case class Activation(properties: Seq[(String, Option[String])]) +// Maven-specific case class Profile( id: String, activeByDefault: Option[Boolean], @@ -72,6 +85,7 @@ case class Profile( properties: Map[String, String] ) +// Maven-specific case class Versions( latest: String, release: String, @@ -90,6 +104,7 @@ object Versions { ) } +// Maven-specific case class SnapshotVersion( classifier: String, extension: String, @@ -97,6 +112,7 @@ case class SnapshotVersion( updated: Option[Versions.DateTime] ) +// Maven-specific case class SnapshotVersioning( module: Module, version: String, @@ -109,6 +125,13 @@ case class SnapshotVersioning( snapshotVersions: Seq[SnapshotVersion] ) +// Ivy-specific +case class Publication( + name: String, + `type`: String, + ext: String +) + case class Artifact( url: String, checksumUrls: Map[String, String], diff --git a/core/shared/src/main/scala/coursier/core/Orders.scala b/core/shared/src/main/scala/coursier/core/Orders.scala index 7df4a87eb..2952604df 100644 --- a/core/shared/src/main/scala/coursier/core/Orders.scala +++ b/core/shared/src/main/scala/coursier/core/Orders.scala @@ -115,15 +115,25 @@ object Orders { } } + private def fallbackConfigIfNecessary(dep: Dependency, configs: Set[String]): Dependency = + Parse.withFallbackConfig(dep.configuration) match { + case Some((main, fallback)) if !configs(main) && configs(fallback) => + dep.copy(configuration = fallback) + case _ => + dep + } + /** * Assume all dependencies have same `module`, `version`, and `artifact`; see `minDependencies` * if they don't. */ def minDependenciesUnsafe( dependencies: Set[Dependency], - configs: ((Module, String)) => Map[String, Seq[String]] + configs: Map[String, Seq[String]] ): Set[Dependency] = { + val availableConfigs = configs.keySet val groupedDependencies = dependencies + .map(fallbackConfigIfNecessary(_, availableConfigs)) .groupBy(dep => (dep.optional, dep.configuration)) .mapValues(deps => deps.head.copy(exclusions = deps.foldLeft(Exclusions.one)((acc, dep) => Exclusions.meet(acc, dep.exclusions)))) .toList @@ -132,7 +142,7 @@ object Orders { for { List(((xOpt, xScope), xDep), ((yOpt, yScope), yDep)) <- groupedDependencies.combinations(2) optCmp <- optionalPartialOrder.tryCompare(xOpt, yOpt).iterator - scopeCmp <- configurationPartialOrder(configs(xDep.moduleVersion)).tryCompare(xScope, yScope).iterator + scopeCmp <- configurationPartialOrder(configs).tryCompare(xScope, yScope).iterator if optCmp*scopeCmp >= 0 exclCmp <- exclusionsPartialOrder.tryCompare(xDep.exclusions, yDep.exclusions).iterator if optCmp*exclCmp >= 0 @@ -156,7 +166,7 @@ object Orders { ): Set[Dependency] = { dependencies .groupBy(_.copy(configuration = "", exclusions = Set.empty, optional = false)) - .mapValues(minDependenciesUnsafe(_, configs)) + .mapValues(deps => minDependenciesUnsafe(deps, configs(deps.head.moduleVersion))) .valuesIterator .fold(Set.empty)(_ ++ _) } diff --git a/core/shared/src/main/scala/coursier/core/Parse.scala b/core/shared/src/main/scala/coursier/core/Parse.scala index 3259d1ef1..a813b26f1 100644 --- a/core/shared/src/main/scala/coursier/core/Parse.scala +++ b/core/shared/src/main/scala/coursier/core/Parse.scala @@ -1,5 +1,6 @@ package coursier.core +import java.util.regex.Pattern.quote import coursier.core.compatibility._ object Parse { @@ -31,4 +32,20 @@ object Parse { .orElse(versionInterval(s).map(VersionConstraint.Interval)) } + val fallbackConfigRegex = { + val noPar = "([^" + quote("()") + "]*)" + "^" + noPar + quote("(") + noPar + quote(")") + "$" + }.r + + def withFallbackConfig(config: String): Option[(String, String)] = + Parse.fallbackConfigRegex.findAllMatchIn(config).toSeq match { + case Seq(m) => + assert(m.groupCount == 2) + val main = config.substring(m.start(1), m.end(1)) + val fallback = config.substring(m.start(2), m.end(2)) + Some((main, fallback)) + case _ => + None + } + } diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index 891afee00..89306f1ff 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -286,7 +286,18 @@ object Resolution { helper(extraConfigs, acc ++ configs) } - helper(Set(config), Set.empty) + val config0 = Parse.withFallbackConfig(config) match { + case Some((main, fallback)) => + if (configurations.contains(main)) + main + else if (configurations.contains(fallback)) + fallback + else + main + case None => config + } + + helper(Set(config0), Set.empty) } /** @@ -741,6 +752,16 @@ case class Resolution( .artifacts(dep, proj) } yield artifact + def artifactsByDep: Seq[(Dependency, Artifact)] = + for { + dep <- minDependencies.toSeq + (source, proj) <- projectCache + .get(dep.moduleVersion) + .toSeq + artifact <- source + .artifacts(dep, proj) + } yield dep -> artifact + def errors: Seq[(Dependency, Seq[String])] = for { dep <- dependencies.toSeq diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala new file mode 100644 index 000000000..c49054e60 --- /dev/null +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -0,0 +1,214 @@ +package coursier.ivy + +import coursier.core._ +import scala.annotation.tailrec +import scala.util.matching.Regex +import scalaz._ +import java.util.regex.Pattern.quote + +object IvyRepository { + + val optionalPartRegex = (quote("(") + "[^" + quote("()") + "]*" + quote(")")).r + val variableRegex = (quote("[") + "[^" + quote("[()]") + "]*" + quote("]")).r + + sealed abstract class PatternPart(val effectiveStart: Int, val effectiveEnd: Int) extends Product with Serializable { + require(effectiveStart <= effectiveEnd) + def start = effectiveStart + def end = effectiveEnd + + // FIXME Some kind of validation should be used here, to report all the missing variables, + // not only the first one missing. + def apply(content: String): Map[String, String] => String \/ String + } + object PatternPart { + case class Literal(override val effectiveStart: Int, override val effectiveEnd: Int) extends PatternPart(effectiveStart, effectiveEnd) { + def apply(content: String): Map[String, String] => String \/ String = { + assert(content.length == effectiveEnd - effectiveStart) + val matches = variableRegex.findAllMatchIn(content).toList + + variables => + @tailrec + def helper(idx: Int, matches: List[Regex.Match], b: StringBuilder): String \/ String = + if (idx >= content.length) + \/-(b.result()) + else { + assert(matches.headOption.forall(_.start >= idx)) + matches.headOption.filter(_.start == idx) match { + case Some(m) => + val variableName = content.substring(m.start + 1, m.end - 1) + variables.get(variableName) match { + case None => -\/(s"Variable not found: $variableName") + case Some(value) => + b ++= value + helper(m.end, matches.tail, b) + } + case None => + val nextIdx = matches.headOption.fold(content.length)(_.start) + b ++= content.substring(idx, nextIdx) + helper(nextIdx, matches, b) + } + } + + helper(0, matches, new StringBuilder) + } + } + case class Optional(start0: Int, end0: Int) extends PatternPart(start0 + 1, end0 - 1) { + override def start = start0 + override def end = end0 + + def apply(content: String): Map[String, String] => String \/ String = { + assert(content.length == effectiveEnd - effectiveStart) + val inner = Literal(effectiveStart, effectiveEnd).apply(content) + + variables => + \/-(inner(variables).fold(_ => "", x => x)) + } + } + } + +} + +case class IvyRepository(pattern: String) extends Repository { + + import Repository._ + import IvyRepository._ + + val parts = { + val optionalParts = optionalPartRegex.findAllMatchIn(pattern).toList.map { m => + PatternPart.Optional(m.start, m.end) + } + + val len = pattern.length + + @tailrec + def helper( + idx: Int, + opt: List[PatternPart.Optional], + acc: List[PatternPart] + ): Vector[PatternPart] = + if (idx >= len) + acc.toVector.reverse + else + opt match { + case Nil => + helper(len, Nil, PatternPart.Literal(idx, len) :: acc) + case (opt0 @ PatternPart.Optional(start0, end0)) :: rem => + if (idx < start0) + helper(start0, opt, PatternPart.Literal(idx, start0) :: acc) + else { + assert(idx == start0, s"idx: $idx, start0: $start0") + helper(end0, rem, opt0 :: acc) + } + } + + helper(0, optionalParts, Nil) + } + + assert(pattern.isEmpty == parts.isEmpty) + if (pattern.nonEmpty) { + for ((a, b) <- parts.zip(parts.tail)) + assert(a.end == b.start) + assert(parts.head.start == 0) + assert(parts.last.end == pattern.length) + } + + private val substituteHelpers = parts.map { part => + part(pattern.substring(part.effectiveStart, part.effectiveEnd)) + } + + def substitute(variables: Map[String, String]): String \/ String = + substituteHelpers.foldLeft[String \/ String](\/-("")) { + case (acc0, helper) => + for { + acc <- acc0 + s <- helper(variables) + } yield acc + s + } + + // If attributes are added to `Module`, they should be added here + // See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a + // list of variables that should be supported. + // Some are missing (branch, conf, originalName). + private def variables( + org: String, + name: String, + version: String, + `type`: String, + artifact: String, + ext: String + ) = + Map( + "organization" -> org, + "organisation" -> org, + "orgPath" -> org.replace('.', '/'), + "module" -> name, + "revision" -> version, + "type" -> `type`, + "artifact" -> artifact, + "ext" -> ext + ) + + + val source: Artifact.Source = new Artifact.Source { + def artifacts(dependency: Dependency, project: Project) = + project + .publications + .collect { case (conf, p) if conf == "*" || conf == dependency.configuration => p } + .flatMap { p => + substitute(variables( + dependency.module.organization, + dependency.module.name, + dependency.version, + p.`type`, + p.name, + p.ext + )).toList.map(p -> _) + } + .map { case (p, url) => + Artifact( + url, + Map.empty, + Map.empty, + Attributes(p.`type`, p.ext) + ) + .withDefaultChecksums + .withDefaultSignature + } + } + + + def find[F[_]]( + module: Module, + version: String, + fetch: Repository.Fetch[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, (Artifact.Source, Project)] = { + + val eitherArtifact: String \/ Artifact = + for { + url <- substitute(variables(module.organization, module.name, version, "ivy", "ivy", "xml")) + } yield + Artifact( + url, + Map.empty, + Map.empty, + Attributes("ivy", "") + ) + .withDefaultChecksums + .withDefaultSignature + + for { + artifact <- EitherT(F.point(eitherArtifact)) + ivy <- fetch(artifact) + proj <- EitherT(F.point { + for { + xml <- \/.fromEither(compatibility.xmlParse(ivy)) + _ <- if (xml.label == "ivy-module") \/-(()) else -\/("Module definition not found") + proj <- IvyXml.project(xml) + } yield proj + }) + } yield (source, proj) + } + +} \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala new file mode 100644 index 000000000..617c7f796 --- /dev/null +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -0,0 +1,127 @@ +package coursier.ivy + +import coursier.core._ +import coursier.util.Xml._ + +import scalaz.{ Node => _, _ }, Scalaz._ + +object IvyXml { + + private def info(node: Node): String \/ (Module, String) = + for { + org <- node.attribute("organisation") + name <- node.attribute("module") + version <- node.attribute("revision") + } yield (Module(org, name), version) + + // FIXME Errors are ignored here + private def configurations(node: Node): Seq[(String, Seq[String])] = + node.children + .filter(_.label == "conf") + .flatMap { node => + node.attribute("name").toOption.toSeq.map(_ -> node) + } + .map { case (name, node) => + name -> node.attribute("extends").toOption.toSeq.flatMap(_.split(',')) + } + + // FIXME Errors ignored as above - warnings should be reported at least for anything suspicious + private def dependencies(node: Node): Seq[(String, Dependency)] = + node.children + .filter(_.label == "dependency") + .flatMap { node => + // artifact and include sub-nodes are ignored here + + val excludes = node.children + .filter(_.label == "exclude") + .flatMap { node0 => + val org = node.attribute("org").getOrElse("*") + val name = node.attribute("module").orElse(node.attribute("name")).getOrElse("*") + val confs = node.attribute("conf").toOption.fold(Seq("*"))(_.split(',')) + confs.map(_ -> (org, name)) + } + .groupBy { case (conf, _) => conf } + .map { case (conf, l) => conf -> l.map { case (_, e) => e }.toSet } + + val allConfsExcludes = excludes.getOrElse("*", Set.empty) + + for { + org <- node.attribute("org").toOption.toSeq + name <- node.attribute("name").toOption.toSeq + version <- node.attribute("rev").toOption.toSeq + rawConf <- node.attribute("conf").toOption.toSeq + (fromConf, toConf) <- rawConf.split(',').toSeq.map(_.split("->", 2)).collect { + case Array(from, to) => from -> to + } + } yield fromConf -> Dependency( + Module(org, name), + version, + toConf, + allConfsExcludes ++ excludes.getOrElse(fromConf, Set.empty), + Attributes("jar", ""), // should come from possible artifact nodes + optional = false + ) + } + + private def publications(node: Node): Map[String, Seq[Publication]] = + node.children + .filter(_.label == "artifact") + .flatMap { node0 => + val name = node.attribute("name").getOrElse("") + val type0 = node.attribute("type").getOrElse("jar") + val ext = node.attribute("ext").getOrElse(type0) + val confs = node.attribute("conf").toOption.fold(Seq("*"))(_.split(',')) + confs.map(_ -> Publication(name, type0, ext)) + } + .groupBy { case (conf, _) => conf } + .map { case (conf, l) => conf -> l.map { case (_, p) => p } } + + def project(node: Node): String \/ Project = + for { + infoNode <- node.children + .find(_.label == "info") + .toRightDisjunction("Info not found") + + (module, version) <- info(infoNode) + + dependenciesNodeOpt = node.children + .find(_.label == "dependencies") + + dependencies0 = dependenciesNodeOpt.map(dependencies).getOrElse(Nil) + + configurationsNodeOpt = node.children + .find(_.label == "configurations") + + configurationsOpt = configurationsNodeOpt.map(configurations) + + configurations0 = configurationsOpt.getOrElse(Seq("default" -> Seq.empty[String])) + + publicationsNodeOpt = node.children + .find(_.label == "publications") + + publicationsOpt = publicationsNodeOpt.map(publications) + + } yield + Project( + module, + version, + dependencies0, + configurations0.toMap, + None, + Nil, + Map.empty, + Nil, + None, + None, + if (publicationsOpt.isEmpty) + // no publications node -> default JAR artifact + Seq("*" -> Publication(module.name, "jar", "jar")) + else + // publications node is there -> only its content (if it is empty, no artifacts, + // as per the Ivy manual) + configurations0.flatMap { case (conf, _) => + publicationsOpt.flatMap(_.get(conf)).getOrElse(Nil).map(conf -> _) + } + ) + +} \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index b2b13f3fe..f8f3c7981 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -37,6 +37,7 @@ object MavenRepository { val defaultConfigurations = Map( + "compile" -> Seq.empty, "runtime" -> Seq("compile"), "test" -> Seq("runtime") ) @@ -106,7 +107,7 @@ case class MavenRepository( Attributes("pom", "") ) .withDefaultChecksums - .withDefaultChecksums + .withDefaultSignature Some(artifact) } diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 4cd412d2a..6ad669204 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -7,21 +7,6 @@ import scalaz._ object Pom { import coursier.util.Xml._ - object Text { - def unapply(n: Node): Option[String] = - if (n.isText) Some(n.textContent) - else None - } - - private def text(elem: Node, label: String, description: String) = { - import Scalaz.ToOptionOpsFromOption - - elem.child - .find(_.label == label) - .flatMap(_.child.collectFirst{case Text(t) => t}) - .toRightDisjunction(s"$description not found") - } - def property(elem: Node): String \/ (String, String) = { // Not matching with Text, which fails on scala-js if the property value has xml comments if (elem.isElement) \/-(elem.label -> elem.textContent) @@ -53,9 +38,9 @@ object Pom { scopeOpt = text(node, "scope", "").toOption typeOpt = text(node, "type", "").toOption classifierOpt = text(node, "classifier", "").toOption - xmlExclusions = node.child + xmlExclusions = node.children .find(_.label == "exclusions") - .map(_.child.filter(_.label == "exclusion")) + .map(_.children.filter(_.label == "exclusion")) .getOrElse(Seq.empty) exclusions <- { import Scalaz._ @@ -66,8 +51,8 @@ object Pom { mod, version0, "", - Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), exclusions.map(mod => (mod.organization, mod.name)).toSet, + Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), optional ) } @@ -80,7 +65,7 @@ object Pom { case _ => None } - val properties = node.child + val properties = node.children .filter(_.label == "property") .flatMap{ p => for{ @@ -97,28 +82,28 @@ object Pom { val id = text(node, "id", "Profile ID").getOrElse("") - val xmlActivationOpt = node.child + val xmlActivationOpt = node.children .find(_.label == "activation") val (activeByDefault, activation) = xmlActivationOpt.fold((Option.empty[Boolean], Activation(Nil)))(profileActivation) - val xmlDeps = node.child + val xmlDeps = node.children .find(_.label == "dependencies") - .map(_.child.filter(_.label == "dependency")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) for { deps <- xmlDeps.toList.traverseU(dependency) - xmlDepMgmts = node.child + xmlDepMgmts = node.children .find(_.label == "dependencyManagement") - .flatMap(_.child.find(_.label == "dependencies")) - .map(_.child.filter(_.label == "dependency")) + .flatMap(_.children.find(_.label == "dependencies")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) depMgmts <- xmlDepMgmts.toList.traverseU(dependency) - xmlProperties = node.child + xmlProperties = node.children .find(_.label == "properties") - .map(_.child.collect{case elem if elem.isElement => elem}) + .map(_.children.collect{case elem if elem.isElement => elem}) .getOrElse(Seq.empty) properties <- { @@ -136,7 +121,7 @@ object Pom { projModule <- module(pom, groupIdIsOptional = true) projVersion = readVersion(pom) - parentOpt = pom.child + parentOpt = pom.children .find(_.label == "parent") parentModuleOpt <- parentOpt .map(module(_).map(Some(_))) @@ -144,16 +129,16 @@ object Pom { parentVersionOpt = parentOpt .map(readVersion) - xmlDeps = pom.child + xmlDeps = pom.children .find(_.label == "dependencies") - .map(_.child.filter(_.label == "dependency")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) deps <- xmlDeps.toList.traverseU(dependency) - xmlDepMgmts = pom.child + xmlDepMgmts = pom.children .find(_.label == "dependencyManagement") - .flatMap(_.child.find(_.label == "dependencies")) - .map(_.child.filter(_.label == "dependency")) + .flatMap(_.children.find(_.label == "dependencies")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) depMgmts <- xmlDepMgmts.toList.traverseU(dependency) @@ -171,15 +156,15 @@ object Pom { .map(mod => if (mod.organization.isEmpty) -\/("Parent organization missing") else \/-(())) .getOrElse(\/-(())) - xmlProperties = pom.child + xmlProperties = pom.children .find(_.label == "properties") - .map(_.child.collect{case elem if elem.isElement => elem}) + .map(_.children.collect{case elem if elem.isElement => elem}) .getOrElse(Seq.empty) properties <- xmlProperties.toList.traverseU(property) - xmlProfiles = pom.child + xmlProfiles = pom.children .find(_.label == "profiles") - .map(_.child.filter(_.label == "profile")) + .map(_.children.filter(_.label == "profile")) .getOrElse(Seq.empty) profiles <- xmlProfiles.toList.traverseU(profile) @@ -187,13 +172,14 @@ object Pom { projModule.copy(organization = groupId), version, deps, + Map.empty, parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), depMgmts, - Map.empty, properties.toMap, profiles, None, - None + None, + Nil ) } @@ -217,7 +203,7 @@ object Pom { organization <- text(node, "groupId", "Organization") // Ignored name <- text(node, "artifactId", "Name") // Ignored - xmlVersioning <- node.child + xmlVersioning <- node.children .find(_.label == "versioning") .toRightDisjunction("Versioning info not found in metadata") @@ -226,9 +212,9 @@ object Pom { release = text(xmlVersioning, "release", "Release version") .getOrElse("") - versionsOpt = xmlVersioning.child + versionsOpt = xmlVersioning.children .find(_.label == "versions") - .map(_.child.filter(_.label == "version").flatMap(_.child.collectFirst{case Text(t) => t})) + .map(_.children.filter(_.label == "version").flatMap(_.children.collectFirst{case Text(t) => t})) lastUpdatedOpt = text(xmlVersioning, "lastUpdated", "Last update date and time") .toOption @@ -268,7 +254,7 @@ object Pom { name <- text(node, "artifactId", "Name") version = readVersion(node) - xmlVersioning <- node.child + xmlVersioning <- node.children .find(_.label == "versioning") .toRightDisjunction("Versioning info not found in metadata") @@ -277,15 +263,15 @@ object Pom { release = text(xmlVersioning, "release", "Release version") .getOrElse("") - versionsOpt = xmlVersioning.child + versionsOpt = xmlVersioning.children .find(_.label == "versions") - .map(_.child.filter(_.label == "version").flatMap(_.child.collectFirst{case Text(t) => t})) + .map(_.children.filter(_.label == "version").flatMap(_.children.collectFirst{case Text(t) => t})) lastUpdatedOpt = text(xmlVersioning, "lastUpdated", "Last update date and time") .toOption .flatMap(parseDateTime) - xmlSnapshotOpt = xmlVersioning.child + xmlSnapshotOpt = xmlVersioning.children .find(_.label == "snapshot") timestamp = xmlSnapshotOpt @@ -313,9 +299,9 @@ object Pom { case "false" => false } - xmlSnapshotVersions = xmlVersioning.child + xmlSnapshotVersions = xmlVersioning.children .find(_.label == "snapshotVersions") - .map(_.child.filter(_.label == "snapshotVersion")) + .map(_.children.filter(_.label == "snapshotVersion")) .getOrElse(Seq.empty) snapshotVersions <- xmlSnapshotVersions .toList diff --git a/core/shared/src/main/scala/coursier/package.scala b/core/shared/src/main/scala/coursier/package.scala index c815d5dcd..3a7125b49 100644 --- a/core/shared/src/main/scala/coursier/package.scala +++ b/core/shared/src/main/scala/coursier/package.scala @@ -19,8 +19,8 @@ package object coursier { module, version, configuration, - attributes, exclusions, + attributes, optional ) } diff --git a/core/shared/src/main/scala/coursier/util/Xml.scala b/core/shared/src/main/scala/coursier/util/Xml.scala index 5518a58d2..a4e9c86df 100644 --- a/core/shared/src/main/scala/coursier/util/Xml.scala +++ b/core/shared/src/main/scala/coursier/util/Xml.scala @@ -1,14 +1,24 @@ package coursier.util +import scalaz.{\/-, -\/, \/, Scalaz} + object Xml { /** A representation of an XML node/document, with different implementations on JVM and JS */ trait Node { def label: String - def child: Seq[Node] + def attributes: Seq[(String, String)] + def children: Seq[Node] def isText: Boolean def textContent: String def isElement: Boolean + + lazy val attributesMap = attributes.toMap + def attribute(name: String): String \/ String = + attributesMap.get(name) match { + case None => -\/(s"Missing attribute $name") + case Some(value) => \/-(value) + } } object Node { @@ -16,10 +26,26 @@ object Xml { new Node { val isText = false val isElement = false - val child = Nil + val children = Nil val label = "" + val attributes = Nil val textContent = "" } } + object Text { + def unapply(n: Node): Option[String] = + if (n.isText) Some(n.textContent) + else None + } + + def text(elem: Node, label: String, description: String) = { + import Scalaz.ToOptionOpsFromOption + + elem.children + .find(_.label == label) + .flatMap(_.children.collectFirst{case Text(t) => t}) + .toRightDisjunction(s"$description not found") + } + } diff --git a/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala index 833cabbbb..bb8452b79 100644 --- a/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala +++ b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala @@ -194,7 +194,7 @@ object PomParsingTests extends TestSuite { val node = parsed.right.get assert(node.label == "properties") - val children = node.child.collect{case elem if elem.isElement => elem} + val children = node.children.collect{case elem if elem.isElement => elem} val props0 = children.toList.traverseU(Pom.property) assert(props0.isRight) diff --git a/tests/shared/src/test/scala/coursier/test/package.scala b/tests/shared/src/test/scala/coursier/test/package.scala index dc8d31315..83d17cf7d 100644 --- a/tests/shared/src/test/scala/coursier/test/package.scala +++ b/tests/shared/src/test/scala/coursier/test/package.scala @@ -13,27 +13,50 @@ package object test { core.Activation(properties) } - def apply(id: String, - activeByDefault: Option[Boolean] = None, - activation: Activation = Activation(), - dependencies: Seq[(String, Dependency)] = Nil, - dependencyManagement: Seq[(String, Dependency)] = Nil, - properties: Map[String, String] = Map.empty) = - core.Profile(id, activeByDefault, activation, dependencies, dependencyManagement, properties) + def apply( + id: String, + activeByDefault: Option[Boolean] = None, + activation: Activation = Activation(), + dependencies: Seq[(String, Dependency)] = Nil, + dependencyManagement: Seq[(String, Dependency)] = Nil, + properties: Map[String, String] = Map.empty + ) = + core.Profile( + id, + activeByDefault, + activation, + dependencies, + dependencyManagement, + properties + ) } object Project { - def apply(module: Module, - version: String, - dependencies: Seq[(String, Dependency)] = Seq.empty, - parent: Option[ModuleVersion] = None, - dependencyManagement: Seq[(String, Dependency)] = Seq.empty, - configurations: Map[String, Seq[String]] = Map.empty, - properties: Map[String, String] = Map.empty, - profiles: Seq[Profile] = Seq.empty, - versions: Option[core.Versions] = None, - snapshotVersioning: Option[core.SnapshotVersioning] = None - ): Project = - core.Project(module, version, dependencies, parent, dependencyManagement, configurations, properties, profiles, versions, snapshotVersioning) + def apply( + module: Module, + version: String, + dependencies: Seq[(String, Dependency)] = Seq.empty, + parent: Option[ModuleVersion] = None, + dependencyManagement: Seq[(String, Dependency)] = Seq.empty, + configurations: Map[String, Seq[String]] = Map.empty, + properties: Map[String, String] = Map.empty, + profiles: Seq[Profile] = Seq.empty, + versions: Option[core.Versions] = None, + snapshotVersioning: Option[core.SnapshotVersioning] = None, + publications: Seq[(String, core.Publication)] = Nil + ): Project = + core.Project( + module, + version, + dependencies, + configurations, + parent, + dependencyManagement, + properties, + profiles, + versions, + snapshotVersioning, + publications + ) } } From f84e9ad9386cfd06a801a365bab1ea3918f9e520 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:32 +0100 Subject: [PATCH 04/80] Changes in files --- .../main/scala/coursier/cli/TermDisplay.scala | 4 +- .../src/main/scala/coursier/Fetch.scala | 82 +++++ .../main/scala/coursier/core/Repository.scala | 50 +-- .../main/scala/coursier/core/Resolution.scala | 59 +++- .../coursier/core/ResolutionProcess.scala | 15 +- .../scala/coursier/ivy/IvyRepository.scala | 3 +- .../coursier/maven/MavenRepository.scala | 12 +- fetch-js/src/main/scala/coursier/Fetch.scala | 26 -- .../src/main/scala/coursier/Platform.scala | 9 +- .../scala/scalaz/concurrent/package.scala | 19 +- files/src/main/scala/coursier/Fetch.scala | 29 -- files/src/main/scala/coursier/Files.scala | 321 +++++++++++------- files/src/main/scala/coursier/Platform.scala | 7 +- .../scala/coursier/test/CentralTests.scala | 2 +- .../scala/coursier/test/ResolutionTests.scala | 2 +- .../scala/coursier/test/TestRepository.scala | 2 +- web/src/main/scala/coursier/web/Backend.scala | 6 +- 17 files changed, 379 insertions(+), 269 deletions(-) create mode 100644 core/shared/src/main/scala/coursier/Fetch.scala delete mode 100644 fetch-js/src/main/scala/coursier/Fetch.scala delete mode 100644 files/src/main/scala/coursier/Fetch.scala diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cli/src/main/scala/coursier/cli/TermDisplay.scala index 62f98adc3..d973da446 100644 --- a/cli/src/main/scala/coursier/cli/TermDisplay.scala +++ b/cli/src/main/scala/coursier/cli/TermDisplay.scala @@ -1,6 +1,6 @@ package coursier.cli -import java.io.Writer +import java.io.{File, Writer} import java.util.concurrent._ import ammonite.terminal.{ TTY, Ansi } @@ -109,7 +109,7 @@ class TermDisplay(out: Writer) extends Logger { q.put(Right(())) } - override def downloadingArtifact(url: String): Unit = { + override def downloadingArtifact(url: String, file: File): Unit = { assert(!infos.containsKey(url)) val prev = infos.putIfAbsent(url, Info(0L, None)) assert(prev == null) diff --git a/core/shared/src/main/scala/coursier/Fetch.scala b/core/shared/src/main/scala/coursier/Fetch.scala new file mode 100644 index 000000000..7bd8a15e9 --- /dev/null +++ b/core/shared/src/main/scala/coursier/Fetch.scala @@ -0,0 +1,82 @@ +package coursier + +import scalaz._ + +object Fetch { + + type Content[F[_]] = Artifact => EitherT[F, String, String] + + + type MD = Seq[( + (Module, String), + Seq[String] \/ (Artifact.Source, Project) + )] + + type Metadata[F[_]] = Seq[(Module, String)] => F[MD] + + /** + * Try to find `module` among `repositories`. + * + * Look at `repositories` from the left, one-by-one, and stop at first success. + * Else, return all errors, in the same order. + * + * The `version` field of the returned `Project` in case of success may not be + * equal to the provided one, in case the latter is not a specific + * version (e.g. version interval). Which version get chosen depends on + * the repository implementation. + */ + def find[F[_]]( + repositories: Seq[Repository], + module: Module, + version: String, + fetch: Content[F] + )(implicit + F: Monad[F] + ): EitherT[F, Seq[String], (Artifact.Source, Project)] = { + + val lookups = repositories + .map(repo => repo -> repo.find(module, version, fetch).run) + + val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { + case (acc, (repo, eitherProjTask)) => + F.bind(acc) { + case -\/(errors) => + F.map(eitherProjTask)(_.flatMap{case (source, project) => + if (project.module == module) \/-((source, project)) + else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") + }.leftMap(error => error +: errors)) + + case res @ \/-(_) => + F.point(res) + } + } + + EitherT(F.map(task)(_.leftMap(_.reverse))) + .map {case x @ (_, proj) => + assert(proj.module == module) + x + } + } + + def apply[F[_]]( + repositories: Seq[core.Repository], + fetch: Content[F], + extra: Content[F]* + )(implicit + F: Nondeterminism[F] + ): Metadata[F] = { + + modVers => + F.map( + F.gatherUnordered( + modVers.map { case (module, version) => + def get(fetch: Content[F]) = + find(repositories, module, version, fetch) + F.map((get(fetch) /: extra)(_ orElse get(_)) + .run)((module, version) -> _) + } + ) + )(_.toSeq) + } + +} \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/core/Repository.scala b/core/shared/src/main/scala/coursier/core/Repository.scala index 69bcfa457..8c853874c 100644 --- a/core/shared/src/main/scala/coursier/core/Repository.scala +++ b/core/shared/src/main/scala/coursier/core/Repository.scala @@ -1,5 +1,7 @@ package coursier.core +import coursier.Fetch + import scala.language.higherKinds import scalaz._ @@ -10,7 +12,7 @@ trait Repository { def find[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, (Artifact.Source, Project)] @@ -18,52 +20,6 @@ trait Repository { object Repository { - type Fetch[F[_]] = Artifact => EitherT[F, String, String] - - /** - * Try to find `module` among `repositories`. - * - * Look at `repositories` from the left, one-by-one, and stop at first success. - * Else, return all errors, in the same order. - * - * The `version` field of the returned `Project` in case of success may not be - * equal to the provided one, in case the latter is not a specific - * version (e.g. version interval). Which version get chosen depends on - * the repository implementation. - */ - def find[F[_]]( - repositories: Seq[Repository], - module: Module, - version: String, - fetch: Repository.Fetch[F] - )(implicit - F: Monad[F] - ): EitherT[F, Seq[String], (Artifact.Source, Project)] = { - - val lookups = repositories - .map(repo => repo -> repo.find(module, version, fetch).run) - - val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { - case (acc, (repo, eitherProjTask)) => - F.bind(acc) { - case -\/(errors) => - F.map(eitherProjTask)(_.flatMap{case (source, project) => - if (project.module == module) \/-((source, project)) - else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") - }.leftMap(error => error +: errors)) - - case res @ \/-(_) => - F.point(res) - } - } - - EitherT(F.map(task)(_.leftMap(_.reverse))) - .map {case x @ (_, proj) => - assert(proj.module == module) - x - } - } - implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { def withDefaultChecksums: Artifact = underlying.copy(checksumUrls = underlying.checksumUrls ++ Seq( diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index 89306f1ff..f9eb25759 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -165,33 +165,35 @@ object Resolution { def merge( dependencies: TraversableOnce[Dependency], forceVersions: Map[Module, String] - ): (Seq[Dependency], Seq[Dependency]) = { + ): (Seq[Dependency], Seq[Dependency], Map[Module, String]) = { val mergedByModVer = dependencies .toList .groupBy(dep => dep.module) .map { case (module, deps) => module -> { - forceVersions.get(module) match { + val (versionOpt, updatedDeps) = forceVersions.get(module) match { case None => - if (deps.lengthCompare(1) == 0) \/-(deps) + if (deps.lengthCompare(1) == 0) (Some(deps.head.version), \/-(deps)) else { val versions = deps .map(_.version) .distinct val versionOpt = mergeVersions(versions) - versionOpt match { + (versionOpt, versionOpt match { case Some(version) => \/-(deps.map(dep => dep.copy(version = version))) case None => -\/(deps) - } + }) } case Some(forcedVersion) => - \/-(deps.map(dep => dep.copy(version = forcedVersion))) + (Some(forcedVersion), \/-(deps.map(dep => dep.copy(version = forcedVersion)))) } + + (updatedDeps, versionOpt) } } @@ -201,11 +203,13 @@ object Resolution { ( merged - .collect{case -\/(dep) => dep} + .collect { case (-\/(dep), _) => dep } .flatten, merged - .collect{case \/-(dep) => dep} - .flatten + .collect { case (\/-(dep), _) => dep } + .flatten, + mergedByModVer + .collect { case (mod, (_, Some(ver))) => mod -> ver } ) } @@ -449,10 +453,10 @@ case class Resolution( * May contain dependencies added in previous iterations, but no more * required. These are filtered below, see `newDependencies`. * - * Returns a tuple made of the conflicting dependencies, and all - * the dependencies. + * Returns a tuple made of the conflicting dependencies, all + * the dependencies, and the retained version of each module. */ - def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = + def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency], Map[Module, String]) = // TODO Provide the modules whose version was forced by dependency overrides too merge( rootDependencies.map(withDefaultConfig) ++ dependencies ++ transitiveDependencies, @@ -478,7 +482,7 @@ case class Resolution( */ def isDone: Boolean = { def isFixPoint = { - val (nextConflicts, _) = nextDependenciesAndConflicts + val (nextConflicts, _, _) = nextDependenciesAndConflicts dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet @@ -497,7 +501,7 @@ case class Resolution( * The versions of all the dependencies returned are erased (emptied). */ def reverseDependencies: Map[Dependency, Vector[Dependency]] = { - val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts + val (updatedConflicts, updatedDeps, _) = nextDependenciesAndConflicts val trDepsSeq = for { @@ -566,7 +570,7 @@ case class Resolution( } private def nextNoMissingUnsafe: Resolution = { - val (newConflicts, _) = nextDependenciesAndConflicts + val (newConflicts, _, _) = nextDependenciesAndConflicts copy( dependencies = newDependencies ++ newConflicts, @@ -752,7 +756,7 @@ case class Resolution( .artifacts(dep, proj) } yield artifact - def artifactsByDep: Seq[(Dependency, Artifact)] = + def dependencyArtifacts: Seq[(Dependency, Artifact)] = for { dep <- minDependencies.toSeq (source, proj) <- projectCache @@ -769,4 +773,27 @@ case class Resolution( .get(dep.moduleVersion) .toSeq } yield (dep, err) + + def part(dependencies: Set[Dependency]): Resolution = { + val (_, _, finalVersions) = nextDependenciesAndConflicts + + @tailrec def helper(current: Set[Dependency]): Set[Dependency] = { + val newDeps = current ++ current + .flatMap(finalDependencies0) + .map(dep => dep.copy(version = finalVersions.getOrElse(dep.module, dep.version))) + + val anyNewDep = (newDeps -- current).nonEmpty + + if (anyNewDep) + helper(newDeps) + else + newDeps + } + + copy( + rootDependencies = dependencies, + dependencies = helper(dependencies) + // don't know if something should be done about conflicts + ) + } } diff --git a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala index 5e19adf25..640c0e499 100644 --- a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala +++ b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala @@ -7,7 +7,7 @@ import scala.annotation.tailrec sealed trait ResolutionProcess { def run[F[_]]( - fetch: ResolutionProcess.Fetch[F], + fetch: Fetch.Metadata[F], maxIterations: Int = -1 )(implicit F: Monad[F] @@ -34,7 +34,7 @@ sealed trait ResolutionProcess { } def next[F[_]]( - fetch: ResolutionProcess.Fetch[F] + fetch: Fetch.Metadata[F] )(implicit F: Monad[F] ): F[ResolutionProcess] = { @@ -58,7 +58,7 @@ case class Missing( cont: Resolution => ResolutionProcess ) extends ResolutionProcess { - def next(results: ResolutionProcess.FetchResult): ResolutionProcess = { + def next(results: Fetch.MD): ResolutionProcess = { val errors = results .collect{case (modVer, -\/(errs)) => modVer -> errs } @@ -72,7 +72,7 @@ case class Missing( val depMgmtMissing = depMgmtMissing0 -- results.map(_._1) def cont0(res: Resolution) = { - val res0 = + val res0 = successes.foldLeft(res){case (acc, (modVer, (source, proj))) => acc.copy(projectCache = acc.projectCache + ( modVer -> (source, acc.withDependencyManagement(proj)) @@ -121,12 +121,5 @@ object ResolutionProcess { else Missing(resolution0.missingFromCache.toSeq, resolution0, apply) } - - type FetchResult = Seq[( - (Module, String), - Seq[String] \/ (Artifact.Source, Project) - )] - - type Fetch[F[_]] = Seq[(Module, String)] => F[FetchResult] } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index c49054e60..ab0c814c9 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -1,5 +1,6 @@ package coursier.ivy +import coursier.Fetch import coursier.core._ import scala.annotation.tailrec import scala.util.matching.Regex @@ -180,7 +181,7 @@ case class IvyRepository(pattern: String) extends Repository { def find[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, (Artifact.Source, Project)] = { diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index f8f3c7981..6fc0b02f3 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -1,5 +1,6 @@ package coursier.maven +import coursier.Fetch import coursier.core._ import coursier.core.compatibility.encodeURIComponent @@ -39,6 +40,7 @@ object MavenRepository { val defaultConfigurations = Map( "compile" -> Seq.empty, "runtime" -> Seq("compile"), + "default" -> Seq("runtime"), "test" -> Seq("runtime") ) @@ -141,7 +143,7 @@ case class MavenRepository( def versions[F[_]]( module: Module, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, Versions] = @@ -163,7 +165,7 @@ case class MavenRepository( def snapshotVersioning[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, SnapshotVersioning] = { @@ -187,7 +189,7 @@ case class MavenRepository( def findNoInterval[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, Project] = @@ -226,7 +228,7 @@ case class MavenRepository( module: Module, version: String, versioningValue: Option[String], - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, Project] = { @@ -247,7 +249,7 @@ case class MavenRepository( def find[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, (Artifact.Source, Project)] = { diff --git a/fetch-js/src/main/scala/coursier/Fetch.scala b/fetch-js/src/main/scala/coursier/Fetch.scala deleted file mode 100644 index 3e7d220d5..000000000 --- a/fetch-js/src/main/scala/coursier/Fetch.scala +++ /dev/null @@ -1,26 +0,0 @@ -package coursier - -import scalaz.concurrent.Task - -object Fetch { - - implicit def default( - repositories: Seq[core.Repository] - ): ResolutionProcess.Fetch[Task] = - apply(repositories, Platform.artifact) - - def apply( - repositories: Seq[core.Repository], - fetch: Repository.Fetch[Task] - ): ResolutionProcess.Fetch[Task] = { - - modVers => Task.gatherUnordered( - modVers.map { case (module, version) => - Repository.find(repositories, module, version, fetch) - .run - .map((module, version) -> _) - } - ) - } - -} diff --git a/fetch-js/src/main/scala/coursier/Platform.scala b/fetch-js/src/main/scala/coursier/Platform.scala index 799c20fb5..70e386506 100644 --- a/fetch-js/src/main/scala/coursier/Platform.scala +++ b/fetch-js/src/main/scala/coursier/Platform.scala @@ -75,7 +75,7 @@ object Platform { p.future } - val artifact: Repository.Fetch[Task] = { artifact => + val artifact: Fetch.Content[Task] = { artifact => EitherT( Task { implicit ec => get(artifact.url) @@ -87,13 +87,18 @@ object Platform { ) } + implicit def fetch( + repositories: Seq[core.Repository] + ): Fetch.Metadata[Task] = + Fetch(repositories, Platform.artifact) + trait Logger { def fetching(url: String): Unit def fetched(url: String): Unit def other(url: String, msg: String): Unit } - def artifactWithLogger(logger: Logger): Repository.Fetch[Task] = { artifact => + def artifactWithLogger(logger: Logger): Fetch.Content[Task] = { artifact => EitherT( Task { implicit ec => Future(logger.fetching(artifact.url)) diff --git a/fetch-js/src/main/scala/scalaz/concurrent/package.scala b/fetch-js/src/main/scala/scalaz/concurrent/package.scala index 0a8a8591f..65cb276a6 100644 --- a/fetch-js/src/main/scala/scalaz/concurrent/package.scala +++ b/fetch-js/src/main/scala/scalaz/concurrent/package.scala @@ -2,7 +2,11 @@ package scalaz import scala.concurrent.{ ExecutionContext, Future } -/** Minimal Future-based Task */ +/** + * Minimal Future-based Task. + * + * Likely to be flawed, but does the job. + */ package object concurrent { trait Task[T] { self => @@ -32,10 +36,19 @@ package object concurrent { def runF(implicit ec: ExecutionContext) = Future.traverse(tasks)(_.runF) } - implicit val taskMonad: Monad[Task] = - new Monad[Task] { + implicit val taskMonad: Nondeterminism[Task] = + new Nondeterminism[Task] { def point[A](a: => A): Task[A] = Task.now(a) def bind[A,B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f) + override def reduceUnordered[A, M](fs: Seq[Task[A]])(implicit R: Reducer[A, M]): Task[M] = + Task { implicit ec => + val f = Future.sequence(fs.map(_.runF)) + f.map { l => + (R.zero /: l)(R.snoc) + } + } + def chooseAny[A](head: Task[A], tail: Seq[Task[A]]): Task[(A, Seq[Task[A]])] = + ??? } } diff --git a/files/src/main/scala/coursier/Fetch.scala b/files/src/main/scala/coursier/Fetch.scala deleted file mode 100644 index 2c0ab0e81..000000000 --- a/files/src/main/scala/coursier/Fetch.scala +++ /dev/null @@ -1,29 +0,0 @@ -package coursier - -import scalaz.concurrent.Task - -object Fetch { - - implicit def default( - repositories: Seq[core.Repository] - ): ResolutionProcess.Fetch[Task] = - apply(repositories, Platform.artifact) - - def apply( - repositories: Seq[core.Repository], - fetch: Repository.Fetch[Task], - extra: Repository.Fetch[Task]* - ): ResolutionProcess.Fetch[Task] = { - - modVers => Task.gatherUnordered( - modVers.map { case (module, version) => - def get(fetch: Repository.Fetch[Task]) = - Repository.find(repositories, module, version, fetch) - (get(fetch) /: extra)(_ orElse get(_)) - .run - .map((module, version) -> _) - } - ) - } - -} diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index b7e6d7287..e798338d8 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -3,13 +3,13 @@ package coursier import java.net.URL import java.nio.channels.{ OverlappingFileLockException, FileLock } import java.security.MessageDigest -import java.util.concurrent.{ Executors, ExecutorService } +import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService} import scala.annotation.tailrec import scalaz._ import scalaz.concurrent.{ Task, Strategy } -import java.io._ +import java.io.{ Serializable => _, _ } case class Files( cache: Seq[(String, File)], @@ -17,6 +17,8 @@ case class Files( concurrentDownloadCount: Int = Files.defaultConcurrentDownloadCount ) { + import Files.urlLocks + lazy val defaultPool = Executors.newFixedThreadPool(concurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) @@ -54,7 +56,7 @@ case class Files( def download( artifact: Artifact, - withChecksums: Boolean = true, + checksums: Set[String], logger: Option[Files.Logger] = None )(implicit cachePolicy: CachePolicy, @@ -64,90 +66,122 @@ case class Files( .extra .getOrElse("local", artifact) - val pairs = - Seq(artifact0.url -> artifact.url) ++ { - if (withChecksums) - (artifact0.checksumUrls.keySet intersect artifact.checksumUrls.keySet) - .toList - .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) - else - Nil + val checksumPairs = checksums + .intersect(artifact0.checksumUrls.keySet) + .intersect(artifact.checksumUrls.keySet) + .toSeq + .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) + + val pairs = (artifact0.url -> artifact.url) +: checksumPairs + + + def locally(file: File, url: String): EitherT[Task, FileError, File] = + EitherT { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(url, file)) + \/-(file) + } else + -\/(FileError.NotFound(file.toString): FileError) + } } + def downloadIfDifferent(file: File, url: String): EitherT[Task, FileError, Boolean] = { + ??? + } - def locally(file: File, url: String) = - Task { - if (file.exists()) { - logger.foreach(_.foundLocally(url, file)) - \/-(file) - } else - -\/(FileError.NotFound(file.toString): FileError) + def test = { + val t: Task[List[((File, String), FileError \/ Boolean)]] = Nondeterminism[Task].gather(checksumPairs.map { case (file, url) => + val f = new File(file) + downloadIfDifferent(f, url).run.map((f, url) -> _) + }) + + t.map { l => + val noChange = l.nonEmpty && l.forall { case (_, e) => e.exists(x => x) } + + val anyChange = l.exists { case (_, e) => e.exists(x => !x) } + val anyRecoverableError = l.exists { + case (_, -\/(err: FileError.Recoverable)) => true + case _ => false + } } - // FIXME Things can go wrong here and are not properly handled, + } + + // FIXME Things can go wrong here and are possibly not properly handled, // e.g. what if the connection gets closed during the transfer? // (partial file on disk?) - def remote(file: File, url: String) = - Task { - try { - logger.foreach(_.downloadingArtifact(url)) + def remote(file: File, url: String): EitherT[Task, FileError, File] = + EitherT { + Task { + try { + val o = new Object + val prev = urlLocks.putIfAbsent(url, o) + if (prev == null) { + logger.foreach(_.downloadingArtifact(url, file)) - val conn = new URL(url).openConnection() // FIXME Should this be closed? - // Dummy user-agent instead of the default "Java/...", - // so that we are not returned incomplete/erroneous metadata - // (Maven 2 compatibility? - happens for snapshot versioning metadata, - // this is SO FUCKING CRAZY) - conn.setRequestProperty("User-Agent", "") + val r = try { + val conn = new URL(url).openConnection() // FIXME Should this be closed? + // Dummy user-agent instead of the default "Java/...", + // so that we are not returned incomplete/erroneous metadata + // (Maven 2 compatibility? - happens for snapshot versioning metadata, + // this is SO FUCKING CRAZY) + conn.setRequestProperty("User-Agent", "") - for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) - logger.foreach(_.downloadLength(url, len)) + for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) + logger.foreach(_.downloadLength(url, len)) - val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize) + val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize) - val result = - try { - file.getParentFile.mkdirs() - val out = new FileOutputStream(file) - try { - var lock: FileLock = null - try { - lock = out.getChannel.tryLock() - if (lock == null) - -\/(FileError.Locked(file.toString)) - else { - val b = Array.fill[Byte](Files.bufferSize)(0) + val result = try { + file.getParentFile.mkdirs() + val out = new FileOutputStream(file) + try { + var lock: FileLock = null + try { + lock = out.getChannel.tryLock() + if (lock == null) + -\/(FileError.Locked(file)) + else { + val b = Array.fill[Byte](Files.bufferSize)(0) - @tailrec - def helper(count: Long): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - logger.foreach(_.downloadProgress(url, count + read)) - helper(count + read) + @tailrec + def helper(count: Long): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + out.flush() + logger.foreach(_.downloadProgress(url, count + read)) + helper(count + read) + } + } + + helper(0L) + \/-(file) } - } + } catch { case e: OverlappingFileLockException => + -\/(FileError.Locked(file)) + } finally if (lock != null) lock.release() + } finally out.close() + } finally in.close() - helper(0L) - \/-(file) - } - } - catch { - case e: OverlappingFileLockException => - -\/(FileError.Locked(file.toString)) - } - finally if (lock != null) lock.release() - } finally out.close() - } finally in.close() + for (lastModified <- Option(conn.getLastModified).filter(_ > 0L)) + file.setLastModified(lastModified) - for (lastModified <- Option(conn.getLastModified).filter(_ > 0L)) - file.setLastModified(lastModified) - - logger.foreach(_.downloadedArtifact(url, success = true)) - result - } - catch { case e: Exception => - logger.foreach(_.downloadedArtifact(url, success = false)) - -\/(FileError.DownloadError(e.getMessage)) + result + } catch { case e: Exception => + logger.foreach(_.downloadedArtifact(url, success = false)) + throw e + } finally { + urlLocks.remove(url) + } + logger.foreach(_.downloadedArtifact(url, success = true)) + r + } else + -\/(FileError.ConcurrentDownload(url)) + } catch { case e: Exception => + -\/(FileError.DownloadError(e.getMessage)) + } } } @@ -160,8 +194,8 @@ case class Files( assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url") cachePolicy[FileError \/ File]( _.isLeft )( - locally(file, url) )( - _ => remote(file, url) + locally(file, url).run )( + _ => remote(file, url).run ).map(e => (file, url) -> e.map(_ => ())) } else Task { @@ -182,75 +216,115 @@ case class Files( sumType: String )(implicit pool: ExecutorService = defaultPool - ): Task[FileError \/ Unit] = { + ): EitherT[Task, FileError, Unit] = { val artifact0 = withLocal(artifact) .extra .getOrElse("local", artifact) + EitherT { + artifact0.checksumUrls.get(sumType) match { + case Some(sumFile) => + Task { + val sum = scala.io.Source.fromFile(sumFile) + .getLines() + .toStream + .headOption + .mkString + .takeWhile(!_.isSpaceChar) - artifact0.checksumUrls.get(sumType) match { - case Some(sumFile) => - Task { - val sum = scala.io.Source.fromFile(sumFile) - .getLines() - .toStream - .headOption - .mkString - .takeWhile(!_.isSpaceChar) + val f = new File(artifact0.url) + val md = MessageDigest.getInstance(sumType) + val is = new FileInputStream(f) + val res = try { + var lock: FileLock = null + try { + lock = is.getChannel.tryLock(0L, Long.MaxValue, true) + if (lock == null) + -\/(FileError.Locked(f)) + else { + Files.withContent(is, md.update(_, 0, _)) + \/-(()) + } + } + catch { + case e: OverlappingFileLockException => + -\/(FileError.Locked(f)) + } + finally if (lock != null) lock.release() + } finally is.close() - val md = MessageDigest.getInstance(sumType) - val is = new FileInputStream(new File(artifact0.url)) - try Files.withContent(is, md.update(_, 0, _)) - finally is.close() + res.flatMap { _ => + val digest = md.digest() + val calculatedSum = f"${BigInt(1, digest)}%040x" - val digest = md.digest() - val calculatedSum = f"${BigInt(1, digest)}%040x" + if (sum == calculatedSum) + \/-(()) + else + -\/(FileError.WrongChecksum(sumType, calculatedSum, sum, artifact0.url, sumFile)) + } + } - if (sum == calculatedSum) - \/-(()) - else - -\/(FileError.WrongChecksum(sumType, calculatedSum, sum, artifact0.url, sumFile)) - } - - case None => - Task.now(-\/(FileError.ChecksumNotFound(sumType, artifact0.url))) + case None => + Task.now(-\/(FileError.ChecksumNotFound(sumType, artifact0.url))) + } } } def file( artifact: Artifact, - checksum: Option[String] = Some("SHA-1"), + checksums: Seq[Option[String]] = Seq(Some("SHA-1")), logger: Option[Files.Logger] = None )(implicit cachePolicy: CachePolicy, pool: ExecutorService = defaultPool - ): EitherT[Task, FileError, File] = - EitherT { - val res = download(artifact, withChecksums = checksum.nonEmpty, logger = logger).map { - results => - val ((f, _), res) = results.head - res.map(_ => f) - } + ): EitherT[Task, FileError, File] = { + val checksums0 = if (checksums.isEmpty) Seq(None) else checksums - checksum.fold(res) { sumType => - res.flatMap { - case err @ -\/(_) => Task.now(err) - case \/-(f) => - validateChecksum(artifact, sumType) - .map(_.map(_ => f)) + val res = EitherT { + download( + artifact, + checksums = checksums0.collect { case Some(c) => c }.toSet, + logger = logger + ).map { results => + val checksum = checksums0.find { + case None => true + case Some(c) => + artifact.checksumUrls.get(c).exists { cUrl => + results.exists { case ((_, u), b) => + u == cUrl && b.isRight + } + } + } + + val ((f, _), res) = results.head + res.flatMap { _ => + checksum match { + case None => + // FIXME All the checksums should be in the error, possibly with their URLs + // from artifact.checksumUrls + -\/(FileError.ChecksumNotFound(checksums0.last.get, "")) + case Some(c) => \/-((f, c)) + } } } } + res.flatMap { + case (f, None) => EitherT(Task.now[FileError \/ File](\/-(f))) + case (f, Some(c)) => + validateChecksum(artifact, c).map(_ => f) + } + } + def fetch( - checksum: Option[String] = Some("SHA-1"), + checksums: Seq[Option[String]] = Seq(Some("SHA-1")), logger: Option[Files.Logger] = None )(implicit cachePolicy: CachePolicy, pool: ExecutorService = defaultPool - ): Repository.Fetch[Task] = { + ): Fetch.Content[Task] = { artifact => - file(artifact, checksum = checksum, logger = logger)(cachePolicy).leftMap(_.message).map { f => + file(artifact, checksums = checksums, logger = logger)(cachePolicy).leftMap(_.message).map { f => // FIXME Catch error here? scala.io.Source.fromFile(f)("UTF-8").mkString } @@ -267,9 +341,11 @@ object Files { val defaultConcurrentDownloadCount = 6 + private val urlLocks = new ConcurrentHashMap[String, Object] + trait Logger { def foundLocally(url: String, f: File): Unit = {} - def downloadingArtifact(url: String): Unit = {} + def downloadingArtifact(url: String, file: File): Unit = {} def downloadLength(url: String, length: Long): Unit = {} def downloadProgress(url: String, downloaded: Long): Unit = {} def downloadedArtifact(url: String, success: Boolean): Unit = {} @@ -315,7 +391,7 @@ object Files { } -sealed trait FileError { +sealed trait FileError extends Product with Serializable { def message: String } @@ -327,9 +403,6 @@ object FileError { case class NotFound(file: String) extends FileError { def message = s"$file: not found" } - case class Locked(file: String) extends FileError { - def message = s"$file: locked" - } case class ChecksumNotFound(sumType: String, file: String) extends FileError { def message = s"$file: $sumType checksum not found" } @@ -337,4 +410,12 @@ object FileError { def message = s"$file: $sumType checksum validation failed" } + sealed trait Recoverable extends FileError + case class Locked(file: File) extends Recoverable { + def message = s"$file: locked" + } + case class ConcurrentDownload(url: String) extends Recoverable { + def message = s"$url: concurrent download" + } + } diff --git a/files/src/main/scala/coursier/Platform.scala b/files/src/main/scala/coursier/Platform.scala index c376fefe3..375372f6b 100644 --- a/files/src/main/scala/coursier/Platform.scala +++ b/files/src/main/scala/coursier/Platform.scala @@ -39,7 +39,7 @@ object Platform { } } - val artifact: Repository.Fetch[Task] = { artifact => + val artifact: Fetch.Content[Task] = { artifact => EitherT { val url = new URL(artifact.url) @@ -53,4 +53,9 @@ object Platform { } } + implicit def fetch( + repositories: Seq[core.Repository] + ): Fetch.Metadata[Task] = + Fetch(repositories, Platform.artifact) + } diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index 7ea36ea21..f8b4b85fc 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -4,7 +4,7 @@ package test import utest._ import scala.async.Async.{ async, await } -import coursier.Fetch.default +import coursier.Platform.fetch import coursier.test.compatibility._ object CentralTests extends TestSuite { diff --git a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala index 627ed288b..123461fc1 100644 --- a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala +++ b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala @@ -17,7 +17,7 @@ object ResolutionTests extends TestSuite { ) = Resolution(deps, filter = filter, forceVersions = forceVersions) .process - .run(Fetch.default(repositories)) + .run(Platform.fetch(repositories)) .runF implicit class ProjectOps(val p: Project) extends AnyVal { diff --git a/tests/shared/src/test/scala/coursier/test/TestRepository.scala b/tests/shared/src/test/scala/coursier/test/TestRepository.scala index 85144d66c..fcbe0030f 100644 --- a/tests/shared/src/test/scala/coursier/test/TestRepository.scala +++ b/tests/shared/src/test/scala/coursier/test/TestRepository.scala @@ -13,7 +13,7 @@ class TestRepository(projects: Map[(Module, String), Project]) extends Repositor def find[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ) = diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 8ae7d05a9..f68fbc07a 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -36,12 +36,12 @@ class Backend($: BackendScope[Unit, State]) { def fetch( repositories: Seq[core.Repository], - fetch: Repository.Fetch[Task] - ): ResolutionProcess.Fetch[Task] = { + fetch: Fetch.Content[Task] + ): Fetch.Metadata[Task] = { modVers => Task.gatherUnordered( modVers.map { case (module, version) => - Repository.find(repositories, module, version, fetch) + Fetch.find(repositories, module, version, fetch) .run .map((module, version) -> _) } From cf8979f2850c9aeafaf8c004ea51a2c5e4846699 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:33 +0100 Subject: [PATCH 05/80] Remove deprecated command --- .../main/scala/coursier/cli/Coursier.scala | 85 ------------------- 1 file changed, 85 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index bc7d2eb8f..162fc545f 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -188,91 +188,6 @@ case class Classpath( } -// TODO: allow removing a repository (with confirmations, etc.) -case class Repository( - @ValueDescription("id:baseUrl") - @ExtraName("a") - add: List[String], - @ExtraName("L") - list: Boolean, - @ExtraName("l") - defaultList: Boolean, - ivyLike: Boolean, - @Recurse - cacheOptions: CacheOptions -) extends CoursierCommand { - - if (add.exists(!_.contains(":"))) { - CaseApp.printUsage[Repository](err = true) - sys.exit(255) - } - - val add0 = add - .map{ s => - val Seq(id, baseUrl) = s.split(":", 2).toSeq - id -> baseUrl - } - - if ( - add0.exists(_._1.contains("/")) || - add0.exists(_._1.startsWith(".")) || - add0.exists(_._1.isEmpty) - ) { - CaseApp.printUsage[Repository](err = true) - sys.exit(255) - } - - - val cache = Cache(new File(cacheOptions.cache)) - - if (cache.cache.exists() && !cache.cache.isDirectory) { - Console.err.println(s"Error: ${cache.cache} not a directory") - sys.exit(1) - } - - if (!cache.cache.exists()) - cache.init(verbose = true) - - val current = cache.list().map(_._1).toSet - - val alreadyAdded = add0 - .map(_._1) - .filter(current) - - if (alreadyAdded.nonEmpty) { - Console.err.println(s"Error: already added: ${alreadyAdded.mkString(", ")}") - sys.exit(1) - } - - for ((id, baseUrl0) <- add0) { - val baseUrl = - if (baseUrl0.endsWith("/")) - baseUrl0 - else - baseUrl0 + "/" - - cache.add(id, baseUrl, ivyLike = ivyLike) - } - - if (defaultList) { - val map = cache.repositoryMap() - - for (id <- cache.default(withNotFound = true)) - map.get(id) match { - case Some(repo) => - println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) - case None => - println(s"$id (not found)") - } - } - - if (list) - for ((id, repo, _) <- cache.list().sortBy(_._1)) { - println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) - } - -} - case class Bootstrap( @ExtraName("M") @ExtraName("main") From e937d251b60dffe5948f0012ce4ceafeb486724b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:33 +0100 Subject: [PATCH 06/80] Remove deprecated class --- files/src/main/scala/coursier/Cache.scala | 134 ---------------------- 1 file changed, 134 deletions(-) delete mode 100644 files/src/main/scala/coursier/Cache.scala diff --git a/files/src/main/scala/coursier/Cache.scala b/files/src/main/scala/coursier/Cache.scala deleted file mode 100644 index 4f39c35e9..000000000 --- a/files/src/main/scala/coursier/Cache.scala +++ /dev/null @@ -1,134 +0,0 @@ -package coursier - -import java.io.{ File, PrintWriter } -import scala.io.Source - -object Cache { - - def mavenRepository(lines: Seq[String]): Option[MavenRepository] = { - def isMaven = - lines - .find(_.startsWith("maven:")) - .map(_.stripPrefix("maven:").trim) - .toSeq - .contains("true") - - def ivyLike = - lines - .find(_.startsWith("ivy-like:")) - .map(_.stripPrefix("ivy-like:").trim) - .toSeq - .contains("true") - - def base = - lines - .find(_.startsWith("base:")) - .map(_.stripPrefix("base:").trim) - .filter(_.nonEmpty) - - if (isMaven) - base.map(MavenRepository(_, ivyLike = ivyLike)) - else - None - } - - lazy val default = Cache(new File(sys.props("user.home") + "/.coursier/cache")) - -} - -case class Cache(cache: File) { - - import Cache._ - - lazy val repoDir = new File(cache, "repositories") - lazy val fileBase = new File(cache, "cache") - - lazy val defaultFile = new File(repoDir, ".default") - - def add(id: String, base: String, ivyLike: Boolean): Unit = { - repoDir.mkdirs() - val f = new File(repoDir, id) - val w = new PrintWriter(f) - try w.println((Seq("maven: true", s"base: $base") ++ (if (ivyLike) Seq("ivy-like: true") else Nil)).mkString("\n")) - finally w.close() - } - - def addCentral(): Unit = - add("central", "https://repo1.maven.org/maven2/", ivyLike = false) - - def addIvy2Local(): Unit = - add("ivy2local", new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, ivyLike = true) - - def init( - ifEmpty: Boolean = true, - verbose: Boolean = false - ): Unit = - if (!ifEmpty || !repoDir.exists() || !fileBase.exists()) { - if (verbose) - Console.err.println(s"Initializing $cache") - repoDir.mkdirs() - fileBase.mkdirs() - addCentral() - addIvy2Local() - setDefault("ivy2local", "central") - } - - def setDefault(ids: String*): Unit = { - defaultFile.getParentFile.mkdirs() - val w = new PrintWriter(defaultFile) - try w.println(ids.mkString("\n")) - finally w.close() - } - - def list(): Seq[(String, MavenRepository, (String, File))] = - Option(repoDir.listFiles()) - .map(_.toSeq) - .getOrElse(Nil) - .filter(f => f.isFile && !f.getName.startsWith(".")) - .flatMap { f => - val name = f.getName - val lines = Source.fromFile(f).getLines().toList - mavenRepository(lines).map(repo => - (name, repo, (repo.root, new File(fileBase, name))) - ) - } - - def map(): Map[String, (MavenRepository, (String, File))] = - list() - .map{case (id, repo, fileCache) => id -> (repo, fileCache) } - .toMap - - - def repositories(): Seq[MavenRepository] = - list().map(_._2) - - def repositoryMap(): Map[String, MavenRepository] = - list() - .map{case (id, repo, _) => id -> repo} - .toMap - - def fileCaches(): Seq[(String, File)] = - list().map(_._3) - - def default(withNotFound: Boolean = false): Seq[String] = - if (defaultFile.exists()) { - val default0 = - Source.fromFile(defaultFile) - .getLines() - .map(_.trim) - .filter(_.nonEmpty) - .toList - - val found = list() - .map(_._1) - .toSet - - default0 - .filter(found) - } else - Nil - - def files(): Files = - new Files(list().map{case (_, _, matching) => matching }, () => ???) - -} From 91d4ad0d18b13cb8af1611fc904b865ec6a9c407 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:33 +0100 Subject: [PATCH 07/80] Move scala js custom Task to coursier namespace --- .../src/main/scala/coursier/Platform.scala | 1 - fetch-js/src/main/scala/coursier/Task.scala | 52 ++++++++++++++++++ .../scala/scalaz/concurrent/package.scala | 55 ------------------- web/src/main/scala/coursier/web/Backend.scala | 1 - 4 files changed, 52 insertions(+), 57 deletions(-) create mode 100644 fetch-js/src/main/scala/coursier/Task.scala delete mode 100644 fetch-js/src/main/scala/scalaz/concurrent/package.scala diff --git a/fetch-js/src/main/scala/coursier/Platform.scala b/fetch-js/src/main/scala/coursier/Platform.scala index 70e386506..a9e412e33 100644 --- a/fetch-js/src/main/scala/coursier/Platform.scala +++ b/fetch-js/src/main/scala/coursier/Platform.scala @@ -8,7 +8,6 @@ import scala.scalajs.js import js.Dynamic.{ global => g } import scala.scalajs.js.timers._ -import scalaz.concurrent.Task import scalaz.{ -\/, \/-, EitherT } object Platform { diff --git a/fetch-js/src/main/scala/coursier/Task.scala b/fetch-js/src/main/scala/coursier/Task.scala new file mode 100644 index 000000000..b1ab3aae9 --- /dev/null +++ b/fetch-js/src/main/scala/coursier/Task.scala @@ -0,0 +1,52 @@ +package coursier + +import scala.concurrent.{ ExecutionContext, Future } +import scalaz.{ Nondeterminism, Reducer } + +/** + * Minimal Future-based Task. + * + * Likely to be flawed and/or sub-optimal, but does the job. + */ +trait Task[T] { self => + def map[U](f: T => U): Task[U] = + new Task[U] { + def runF(implicit ec: ExecutionContext) = self.runF.map(f) + } + def flatMap[U](f: T => Task[U]): Task[U] = + new Task[U] { + def runF(implicit ec: ExecutionContext) = self.runF.flatMap(f(_).runF) + } + + def runF(implicit ec: ExecutionContext): Future[T] +} + +object Task { + def now[A](a: A): Task[A] = + new Task[A] { + def runF(implicit ec: ExecutionContext) = Future.successful(a) + } + def apply[A](f: ExecutionContext => Future[A]): Task[A] = + new Task[A] { + def runF(implicit ec: ExecutionContext) = f(ec) + } + def gatherUnordered[T](tasks: Seq[Task[T]], exceptionCancels: Boolean = false): Task[Seq[T]] = + new Task[Seq[T]] { + def runF(implicit ec: ExecutionContext) = Future.traverse(tasks)(_.runF) + } + + implicit val taskMonad: Nondeterminism[Task] = + new Nondeterminism[Task] { + def point[A](a: => A): Task[A] = Task.now(a) + def bind[A,B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f) + override def reduceUnordered[A, M](fs: Seq[Task[A]])(implicit R: Reducer[A, M]): Task[M] = + Task { implicit ec => + val f = Future.sequence(fs.map(_.runF)) + f.map { l => + (R.zero /: l)(R.snoc) + } + } + def chooseAny[A](head: Task[A], tail: Seq[Task[A]]): Task[(A, Seq[Task[A]])] = + ??? + } +} diff --git a/fetch-js/src/main/scala/scalaz/concurrent/package.scala b/fetch-js/src/main/scala/scalaz/concurrent/package.scala deleted file mode 100644 index 65cb276a6..000000000 --- a/fetch-js/src/main/scala/scalaz/concurrent/package.scala +++ /dev/null @@ -1,55 +0,0 @@ -package scalaz - -import scala.concurrent.{ ExecutionContext, Future } - -/** - * Minimal Future-based Task. - * - * Likely to be flawed, but does the job. - */ -package object concurrent { - - trait Task[T] { self => - def map[U](f: T => U): Task[U] = - new Task[U] { - def runF(implicit ec: ExecutionContext) = self.runF.map(f) - } - def flatMap[U](f: T => Task[U]): Task[U] = - new Task[U] { - def runF(implicit ec: ExecutionContext) = self.runF.flatMap(f(_).runF) - } - - def runF(implicit ec: ExecutionContext): Future[T] - } - - object Task { - def now[A](a: A): Task[A] = - new Task[A] { - def runF(implicit ec: ExecutionContext) = Future.successful(a) - } - def apply[A](f: ExecutionContext => Future[A]): Task[A] = - new Task[A] { - def runF(implicit ec: ExecutionContext) = f(ec) - } - def gatherUnordered[T](tasks: Seq[Task[T]], exceptionCancels: Boolean = false): Task[Seq[T]] = - new Task[Seq[T]] { - def runF(implicit ec: ExecutionContext) = Future.traverse(tasks)(_.runF) - } - - implicit val taskMonad: Nondeterminism[Task] = - new Nondeterminism[Task] { - def point[A](a: => A): Task[A] = Task.now(a) - def bind[A,B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f) - override def reduceUnordered[A, M](fs: Seq[Task[A]])(implicit R: Reducer[A, M]): Task[M] = - Task { implicit ec => - val f = Future.sequence(fs.map(_.runF)) - f.map { l => - (R.zero /: l)(R.snoc) - } - } - def chooseAny[A](head: Task[A], tail: Seq[Task[A]]): Task[(A, Seq[Task[A]])] = - ??? - } - } - -} diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index f68fbc07a..5a03d9b97 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -11,7 +11,6 @@ import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue import org.scalajs.jquery.jQuery import scala.concurrent.Future -import scalaz.concurrent.Task import scala.scalajs.js import js.Dynamic.{ global => g } From 46732be5c91f9832d562f96ab3d3deb4fb34a0c4 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:33 +0100 Subject: [PATCH 08/80] Changes in cache policy --- .../main/scala/coursier/cli/Coursier.scala | 10 +- cli/src/main/scala/coursier/cli/Helper.scala | 67 +++--- .../scala/coursier/core/Definitions.scala | 3 +- .../main/scala/coursier/core/Repository.scala | 24 +- .../scala/coursier/ivy/IvyRepository.scala | 8 +- .../coursier/maven/MavenRepository.scala | 12 +- .../scala/coursier/maven/MavenSource.scala | 27 ++- .../src/main/scala/coursier/CachePolicy.scala | 50 +---- files/src/main/scala/coursier/Files.scala | 210 +++++++++++------- 9 files changed, 241 insertions(+), 170 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 162fc545f..d70447079 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -12,12 +12,10 @@ import coursier.util.ClasspathFilter case class CommonOptions( @HelpMessage("Keep optional dependencies (Maven)") keepOptional: Boolean, - @HelpMessage("Off-line mode: only use cache and local repositories") - @ExtraName("c") - offline: Boolean, - @HelpMessage("Force download: for remote repositories only: re-download items, that is, don't use cache directly") - @ExtraName("f") - force: Boolean, + @HelpMessage("Download mode (default: missing, that is fetch things missing from cache)") + @ValueDescription("offline|update-changing|update|missing|force") + @ExtraName("m") + mode: String, @HelpMessage("Quiet output") @ExtraName("q") quiet: Boolean, diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 459d32f84..20371b125 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -2,7 +2,6 @@ package coursier package cli import java.io.{ OutputStreamWriter, File } -import java.util.UUID import coursier.ivy.IvyRepository @@ -10,22 +9,6 @@ import scalaz.{ \/-, -\/ } import scalaz.concurrent.Task object Helper { - def validate(common: CommonOptions) = { - import common._ - - if (force && offline) { - Console.err.println("Error: --offline (-c) and --force (-f) options can't be specified at the same time.") - sys.exit(255) - } - - if (parallel <= 0) { - Console.err.println(s"Error: invalid --parallel (-n) value: $parallel") - sys.exit(255) - } - - ??? - } - def fileRepr(f: File) = f.toString def errPrintln(s: String) = Console.err.println(s) @@ -56,13 +39,23 @@ class Helper( import common._ import Helper.errPrintln - implicit val cachePolicy = - if (offline) - CachePolicy.LocalOnly - else if (force) - CachePolicy.ForceDownload - else - CachePolicy.Default + val cachePolicies = mode match { + case "offline" => + Seq(CachePolicy.LocalOnly) + case "update-changing" => + Seq(CachePolicy.UpdateChanging) + case "update" => + Seq(CachePolicy.Update) + case "missing" => + Seq(CachePolicy.FetchMissing) + case "force" => + Seq(CachePolicy.ForceDownload) + case "default" => + Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing) + case other => + errPrintln(s"Unrecognized mode: $other") + sys.exit(255) + } val files = Files( @@ -200,10 +193,14 @@ class Helper( else None logger.foreach(_.init()) + + val fetchs = cachePolicies.map(p => + files.fetch(logger = logger)(cachePolicy = p) + ) val fetchQuiet = coursier.Fetch( repositories, - files.fetch(logger = logger)(cachePolicy = CachePolicy.LocalOnly), // local files get the priority - files.fetch(logger = logger) + fetchs.head, + fetchs.tail: _* ) val fetch0 = if (verbose0 <= 0) fetchQuiet @@ -296,8 +293,16 @@ class Helper( } def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = { - if (verbose0 >= 0) - errPrintln("Fetching artifacts") + if (verbose0 >= 0) { + val msg = cachePolicies match { + case Seq(CachePolicy.LocalOnly) => + "Checking artifacts" + case _ => + "Fetching artifacts" + } + + errPrintln(msg) + } val artifacts0 = res.artifacts val main0 = main || (!sources && !javadoc) val artifacts = artifacts0.flatMap{ artifact => @@ -318,7 +323,11 @@ class Helper( else None logger.foreach(_.init()) - val tasks = artifacts.map(artifact => files.file(artifact, logger = logger).run.map(artifact.->)) + val tasks = artifacts.map(artifact => + (files.file(artifact, logger = logger)(cachePolicy = cachePolicies.head) /: cachePolicies.tail)( + _ orElse files.file(artifact, logger = logger)(_) + ).run.map(artifact.->) + ) def printTask = Task { if (verbose0 >= 1 && artifacts.nonEmpty) println(s"Found ${artifacts.length} artifacts") diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index e9b52a889..82db9ae31 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -136,7 +136,8 @@ case class Artifact( url: String, checksumUrls: Map[String, String], extra: Map[String, Artifact], - attributes: Attributes + attributes: Attributes, + changing: Boolean ) object Artifact { diff --git a/core/shared/src/main/scala/coursier/core/Repository.scala b/core/shared/src/main/scala/coursier/core/Repository.scala index 8c853874c..2d0f96dcf 100644 --- a/core/shared/src/main/scala/coursier/core/Repository.scala +++ b/core/shared/src/main/scala/coursier/core/Repository.scala @@ -29,16 +29,34 @@ object Repository { def withDefaultSignature: Artifact = underlying.copy(extra = underlying.extra ++ Seq( "sig" -> - Artifact(underlying.url + ".asc", Map.empty, Map.empty, Attributes("asc", "")) + Artifact( + underlying.url + ".asc", + Map.empty, + Map.empty, + Attributes("asc", ""), + changing = underlying.changing + ) .withDefaultChecksums )) def withJavadocSources: Artifact = { val base = underlying.url.stripSuffix(".jar") underlying.copy(extra = underlying.extra ++ Seq( - "sources" -> Artifact(base + "-sources.jar", Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? + "sources" -> Artifact( + base + "-sources.jar", + Map.empty, + Map.empty, + Attributes("jar", "src"), // Are these the right attributes? + changing = underlying.changing + ) .withDefaultChecksums .withDefaultSignature, - "javadoc" -> Artifact(base + "-javadoc.jar", Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above + "javadoc" -> Artifact( + base + "-javadoc.jar", + Map.empty, + Map.empty, + Attributes("jar", "javadoc"), // Same comment as above + changing = underlying.changing + ) .withDefaultChecksums .withDefaultSignature )) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index ab0c814c9..0b496a5c0 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -69,7 +69,7 @@ object IvyRepository { } -case class IvyRepository(pattern: String) extends Repository { +case class IvyRepository(pattern: String, changing: Option[Boolean] = None) extends Repository { import Repository._ import IvyRepository._ @@ -170,7 +170,8 @@ case class IvyRepository(pattern: String) extends Repository { url, Map.empty, Map.empty, - Attributes(p.`type`, p.ext) + Attributes(p.`type`, p.ext), + changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable ) .withDefaultChecksums .withDefaultSignature @@ -194,7 +195,8 @@ case class IvyRepository(pattern: String) extends Repository { url, Map.empty, Map.empty, - Attributes("ivy", "") + Attributes("ivy", ""), + changing = changing.getOrElse(version.contains("-SNAPSHOT")) ) .withDefaultChecksums .withDefaultSignature diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 6fc0b02f3..d0522f13e 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -48,7 +48,8 @@ object MavenRepository { case class MavenRepository( root: String, - ivyLike: Boolean = false + ivyLike: Boolean = false, + changing: Option[Boolean] = None ) extends Repository { import Repository._ @@ -85,7 +86,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = changing.getOrElse(version.contains("-SNAPSHOT")) ) .withDefaultChecksums .withDefaultSignature @@ -106,7 +108,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = true ) .withDefaultChecksums .withDefaultSignature @@ -133,7 +136,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = true ) .withDefaultChecksums .withDefaultSignature diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 02c780d3f..26973836d 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -2,7 +2,12 @@ package coursier.maven import coursier.core._ -case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { +case class MavenSource( + root: String, + ivyLike: Boolean, + changing: Option[Boolean] = None +) extends Artifact.Source { + import Repository._ import MavenRepository._ @@ -39,12 +44,14 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { ) } + val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT")) var artifact = Artifact( root + path.mkString("/"), Map.empty, Map.empty, - dependency.attributes + dependency.attributes, + changing = changing0 ) .withDefaultChecksums @@ -62,10 +69,22 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { artifact .copy( extra = artifact.extra ++ Map( - "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? + "sources" -> Artifact( + srcPath, + Map.empty, + Map.empty, + Attributes("jar", "src"), // Are these the right attributes? + changing = changing0 + ) .withDefaultChecksums .withDefaultSignature, - "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above + "javadoc" -> Artifact( + javadocPath, + Map.empty, + Map.empty, + Attributes("jar", "javadoc"), // Same comment as above + changing = changing0 + ) .withDefaultChecksums .withDefaultSignature )) diff --git a/files/src/main/scala/coursier/CachePolicy.scala b/files/src/main/scala/coursier/CachePolicy.scala index 10c76392f..0bf51e6e3 100644 --- a/files/src/main/scala/coursier/CachePolicy.scala +++ b/files/src/main/scala/coursier/CachePolicy.scala @@ -1,49 +1,11 @@ package coursier -import scalaz.\/ -import scalaz.concurrent.Task - -sealed trait CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] -} +sealed trait CachePolicy extends Product with Serializable object CachePolicy { - def saving[E,T]( - remote: => Task[E \/ T] )( - save: T => Task[Unit] - ): Task[E \/ T] = { - for { - res <- remote - _ <- res.fold(_ => Task.now(()), t => save(t)) - } yield res - } - - case object Default extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res)) - } - case object LocalOnly extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local - } - case object ForceDownload extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - remote(None) - } + case object LocalOnly extends CachePolicy + case object UpdateChanging extends CachePolicy + case object Update extends CachePolicy + case object FetchMissing extends CachePolicy + case object ForceDownload extends CachePolicy } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index e798338d8..d6e5c9eb8 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -1,6 +1,6 @@ package coursier -import java.net.URL +import java.net.{HttpURLConnection, URL} import java.nio.channels.{ OverlappingFileLockException, FileLock } import java.security.MessageDigest import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService} @@ -66,13 +66,24 @@ case class Files( .extra .getOrElse("local", artifact) - val checksumPairs = checksums - .intersect(artifact0.checksumUrls.keySet) - .intersect(artifact.checksumUrls.keySet) - .toSeq - .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) + val pairs = + Seq(artifact0.url -> artifact.url) ++ { + checksums + .intersect(artifact0.checksumUrls.keySet) + .intersect(artifact.checksumUrls.keySet) + .toSeq + .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) + } - val pairs = (artifact0.url -> artifact.url) +: checksumPairs + def urlConn(url: String) = { + val conn = new URL(url).openConnection() // FIXME Should this be closed? + // Dummy user-agent instead of the default "Java/...", + // so that we are not returned incomplete/erroneous metadata + // (Maven 2 compatibility? - happens for snapshot versioning metadata, + // this is SO FUCKING CRAZY) + conn.setRequestProperty("User-Agent", "") + conn + } def locally(file: File, url: String): EitherT[Task, FileError, File] = @@ -86,32 +97,57 @@ case class Files( } } - def downloadIfDifferent(file: File, url: String): EitherT[Task, FileError, Boolean] = { - ??? - } - - def test = { - val t: Task[List[((File, String), FileError \/ Boolean)]] = Nondeterminism[Task].gather(checksumPairs.map { case (file, url) => - val f = new File(file) - downloadIfDifferent(f, url).run.map((f, url) -> _) - }) - - t.map { l => - val noChange = l.nonEmpty && l.forall { case (_, e) => e.exists(x => x) } - - val anyChange = l.exists { case (_, e) => e.exists(x => !x) } - val anyRecoverableError = l.exists { - case (_, -\/(err: FileError.Recoverable)) => true - case _ => false + def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] = + EitherT { + Task { + \/- { + val lastModified = file.lastModified() + if (lastModified > 0L) + Some(lastModified) + else + None + } : FileError \/ Option[Long] } } - } + def urlLastModified(url: String): EitherT[Task, FileError, Option[Long]] = + EitherT { + Task { + urlConn(url) match { + case c: HttpURLConnection => + c.setRequestMethod("HEAD") + val remoteLastModified = c.getLastModified - // FIXME Things can go wrong here and are possibly not properly handled, + \/- { + if (remoteLastModified > 0L) + Some(remoteLastModified) + else + None + } + + case other => + -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)")) + } + } + } + + def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] = + for { + fileLastModOpt <- fileLastModified(file) + urlLastModOpt <- urlLastModified(url) + } yield { + val fromDatesOpt = for { + fileLastMod <- fileLastModOpt + urlLastMod <- urlLastModOpt + } yield fileLastMod < urlLastMod + + fromDatesOpt.getOrElse(true) + } + + // FIXME Things can go wrong here and are not properly handled, // e.g. what if the connection gets closed during the transfer? // (partial file on disk?) - def remote(file: File, url: String): EitherT[Task, FileError, File] = + def remote(file: File, url: String): EitherT[Task, FileError, Unit] = EitherT { Task { try { @@ -121,91 +157,113 @@ case class Files( logger.foreach(_.downloadingArtifact(url, file)) val r = try { - val conn = new URL(url).openConnection() // FIXME Should this be closed? - // Dummy user-agent instead of the default "Java/...", - // so that we are not returned incomplete/erroneous metadata - // (Maven 2 compatibility? - happens for snapshot versioning metadata, - // this is SO FUCKING CRAZY) - conn.setRequestProperty("User-Agent", "") + val conn = urlConn(url) for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) logger.foreach(_.downloadLength(url, len)) val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize) - val result = try { - file.getParentFile.mkdirs() - val out = new FileOutputStream(file) + val result = try { - var lock: FileLock = null + file.getParentFile.mkdirs() + val out = new FileOutputStream(file) try { - lock = out.getChannel.tryLock() - if (lock == null) - -\/(FileError.Locked(file)) - else { - val b = Array.fill[Byte](Files.bufferSize)(0) + var lock: FileLock = null + try { + lock = out.getChannel.tryLock() + if (lock == null) + -\/(FileError.Locked(file)) + else { + val b = Array.fill[Byte](Files.bufferSize)(0) - @tailrec - def helper(count: Long): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - out.flush() - logger.foreach(_.downloadProgress(url, count + read)) - helper(count + read) + @tailrec + def helper(count: Long): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + out.flush() + logger.foreach(_.downloadProgress(url, count + read)) + helper(count + read) + } } - } - helper(0L) - \/-(file) + helper(0L) + \/-(()) + } } - } catch { case e: OverlappingFileLockException => - -\/(FileError.Locked(file)) - } finally if (lock != null) lock.release() - } finally out.close() - } finally in.close() + catch { + case e: OverlappingFileLockException => + -\/(FileError.Locked(file)) + } + finally if (lock != null) lock.release() + } finally out.close() + } finally in.close() for (lastModified <- Option(conn.getLastModified).filter(_ > 0L)) file.setLastModified(lastModified) result - } catch { case e: Exception => + } + catch { case e: Exception => logger.foreach(_.downloadedArtifact(url, success = false)) throw e - } finally { + } + finally { urlLocks.remove(url) } logger.foreach(_.downloadedArtifact(url, success = true)) r } else -\/(FileError.ConcurrentDownload(url)) - } catch { case e: Exception => + } + catch { case e: Exception => -\/(FileError.DownloadError(e.getMessage)) } } } + def checkFileExists(file: File, url: String): EitherT[Task, FileError, Unit] = + EitherT { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(url, file)) + \/-(()) + } else + -\/(FileError.NotFound(file.toString)) + } + } val tasks = for ((f, url) <- pairs) yield { val file = new File(f) - if (url != ("file:" + f) && url != ("file://" + f)) { - assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url") - cachePolicy[FileError \/ File]( - _.isLeft )( - locally(file, url).run )( - _ => remote(file, url).run - ).map(e => (file, url) -> e.map(_ => ())) - } else - Task { - (file, url) -> { - if (file.exists()) - \/-(()) - else - -\/(FileError.NotFound(file.toString)) + val isRemote = url != ("file:" + f) && url != ("file://" + f) + val cachePolicy0 = + if (!isRemote) + CachePolicy.LocalOnly + else if (cachePolicy == CachePolicy.UpdateChanging && !artifact.changing) + CachePolicy.FetchMissing + else + cachePolicy + + val res = cachePolicy match { + case CachePolicy.LocalOnly => + checkFileExists(file, url) + case CachePolicy.UpdateChanging | CachePolicy.Update => + shouldDownload(file, url).flatMap { + case true => + remote(file, url) + case false => + EitherT(Task.now(\/-(()) : FileError \/ Unit)) } - } + case CachePolicy.FetchMissing => + checkFileExists(file, url) orElse remote(file, url) + case CachePolicy.ForceDownload => + remote(file, url) + } + + res.run.map((file, url) -> _) } Nondeterminism[Task].gather(tasks) From 842de76ca6570855dc7e8380eedcfced438796c0 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:34 +0100 Subject: [PATCH 09/80] Add SBT plugin --- build.sbt | 17 +- .../main/scala/coursier/cli/TermDisplay.scala | 9 +- .../coursier/core/compatibility/package.scala | 2 + .../scala/coursier/core/Definitions.scala | 4 + .../src/main/scala/coursier/core/Orders.scala | 49 ++-- .../main/scala/coursier/core/Resolution.scala | 7 +- .../scala/coursier/ivy/IvyRepository.scala | 40 ++- .../src/main/scala/coursier/ivy/IvyXml.scala | 8 +- .../main/scala/coursier/CoursierPlugin.scala | 257 ++++++++++++++++++ plugin/src/main/scala/coursier/FromSbt.scala | 132 +++++++++ .../coursier/InterProjectRepository.scala | 50 ++++ plugin/src/main/scala/coursier/Keys.scala | 18 ++ .../src/main/scala/coursier/Structure.scala | 40 +++ plugin/src/main/scala/coursier/Tasks.scala | 69 +++++ plugin/src/main/scala/coursier/ToSbt.scala | 18 ++ 15 files changed, 672 insertions(+), 48 deletions(-) create mode 100644 plugin/src/main/scala/coursier/CoursierPlugin.scala create mode 100644 plugin/src/main/scala/coursier/FromSbt.scala create mode 100644 plugin/src/main/scala/coursier/InterProjectRepository.scala create mode 100644 plugin/src/main/scala/coursier/Keys.scala create mode 100644 plugin/src/main/scala/coursier/Structure.scala create mode 100644 plugin/src/main/scala/coursier/Tasks.scala create mode 100644 plugin/src/main/scala/coursier/ToSbt.scala diff --git a/build.sbt b/build.sbt index c1a5a3f99..5cdc80b07 100644 --- a/build.sbt +++ b/build.sbt @@ -72,7 +72,6 @@ lazy val commonSettings = baseCommonSettings ++ Seq( } ) - lazy val core = crossProject .settings(commonSettings: _*) .settings(publishingSettings: _*) @@ -163,7 +162,7 @@ lazy val bootstrap = project lazy val cli = project .dependsOn(coreJvm, files) .settings(commonSettings) - .settings(noPublishSettings) + .settings(publishingSettings) .settings(packAutoSettings) .settings( name := "coursier-cli", @@ -171,10 +170,7 @@ lazy val cli = project "com.github.alexarchambault" %% "case-app" % "1.0.0-SNAPSHOT", "com.lihaoyi" %% "ammonite-terminal" % "0.5.0", "ch.qos.logback" % "logback-classic" % "1.1.3" - ), - resourceGenerators in Compile += packageBin.in(bootstrap).in(Compile).map { jar => - Seq(jar) - }.taskValue + ) ) lazy val web = project @@ -208,6 +204,15 @@ lazy val web = project ) ) +// Don't try to compile that if you're not in 2.10 +lazy val plugin = project + .dependsOn(coreJvm, files, cli) + .settings(baseCommonSettings) + .settings( + name := "coursier-sbt-plugin", + sbtPlugin := true + ) + lazy val `coursier` = project.in(file(".")) .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web) .settings(commonSettings) diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cli/src/main/scala/coursier/cli/TermDisplay.scala index d973da446..dee09ea6c 100644 --- a/cli/src/main/scala/coursier/cli/TermDisplay.scala +++ b/cli/src/main/scala/coursier/cli/TermDisplay.scala @@ -26,11 +26,6 @@ class TermDisplay(out: Writer) extends Logger { case Some(Right(())) => // update display - for (_ <- 0 until lineCount) { - ansi.up(1) - ansi.clearLine(2) - } - val downloads0 = downloads.synchronized { downloads .toVector @@ -71,9 +66,13 @@ class TermDisplay(out: Writer) extends Logger { } else (url, extra) + ansi.clearLine(2) out.write(s"$url0 $extra0\n") } + for (_ <- downloads0.indices) + ansi.up(1) + out.flush() Thread.sleep(refreshInterval) helper(downloads0.length) diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index b4874ba94..1b398bcab 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -36,6 +36,8 @@ package object compatibility { def isText = node match { case _: scala.xml.Text => true; case _ => false } def textContent = node.text def isElement = node match { case _: scala.xml.Elem => true; case _ => false } + + override def toString = node.toString } parse.right diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 82db9ae31..8576d835c 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -70,6 +70,10 @@ case class Project( publications: Seq[(String, Publication)] ) { def moduleVersion = (module, version) + + /** All configurations that each configuration extends, including the ones it extends transitively */ + lazy val allConfigurations: Map[String, Set[String]] = + Orders.allConfigurations(configurations) } // Maven-specific diff --git a/core/shared/src/main/scala/coursier/core/Orders.scala b/core/shared/src/main/scala/coursier/core/Orders.scala index 2952604df..d2bd8cdb7 100644 --- a/core/shared/src/main/scala/coursier/core/Orders.scala +++ b/core/shared/src/main/scala/coursier/core/Orders.scala @@ -8,34 +8,39 @@ object Orders { .exists(_ <= 0) } + /** All configurations that each configuration extends, including the ones it extends transitively */ + def allConfigurations(configurations: Map[String, Seq[String]]): Map[String, Set[String]] = { + def allParents(config: String): Set[String] = { + def helper(configs: Set[String], acc: Set[String]): Set[String] = + if (configs.isEmpty) + acc + else if (configs.exists(acc)) + helper(configs -- acc, acc) + else if (configs.exists(!configurations.contains(_))) { + val (remaining, notFound) = configs.partition(configurations.contains) + helper(remaining, acc ++ notFound) + } else { + val extraConfigs = configs.flatMap(configurations) + helper(extraConfigs, acc ++ configs) + } + + helper(Set(config), Set.empty) + } + + configurations + .keys + .toList + .map(config => config -> (allParents(config) - config)) + .toMap + } + /** * Only relations: * Compile < Runtime < Test */ def configurationPartialOrder(configurations: Map[String, Seq[String]]): PartialOrdering[String] = new PartialOrdering[String] { - def allParents(config: String): Set[String] = { - def helper(configs: Set[String], acc: Set[String]): Set[String] = - if (configs.isEmpty) - acc - else if (configs.exists(acc)) - helper(configs -- acc, acc) - else if (configs.exists(!configurations.contains(_))) { - val (remaining, notFound) = configs.partition(configurations.contains) - helper(remaining, acc ++ notFound) - } else { - val extraConfigs = configs.flatMap(configurations) - helper(extraConfigs, acc ++ configs) - } - - helper(Set(config), Set.empty) - } - - val allParentsMap = configurations - .keys - .toList - .map(config => config -> (allParents(config) - config)) - .toMap + val allParentsMap = allConfigurations(configurations) def tryCompare(x: String, y: String) = if (x == y) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index f9eb25759..f2e240bfd 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -777,10 +777,13 @@ case class Resolution( def part(dependencies: Set[Dependency]): Resolution = { val (_, _, finalVersions) = nextDependenciesAndConflicts + def updateVersion(dep: Dependency): Dependency = + dep.copy(version = finalVersions.getOrElse(dep.module, dep.version)) + @tailrec def helper(current: Set[Dependency]): Set[Dependency] = { val newDeps = current ++ current .flatMap(finalDependencies0) - .map(dep => dep.copy(version = finalVersions.getOrElse(dep.module, dep.version))) + .map(updateVersion) val anyNewDep = (newDeps -- current).nonEmpty @@ -792,7 +795,7 @@ case class Resolution( copy( rootDependencies = dependencies, - dependencies = helper(dependencies) + dependencies = helper(dependencies.map(updateVersion)) // don't know if something should be done about conflicts ) } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 0b496a5c0..ac24d534a 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -9,8 +9,9 @@ import java.util.regex.Pattern.quote object IvyRepository { - val optionalPartRegex = (quote("(") + "[^" + quote("()") + "]*" + quote(")")).r - val variableRegex = (quote("[") + "[^" + quote("[()]") + "]*" + quote("]")).r + val optionalPartRegex = (quote("(") + "[^" + quote("{()}") + "]*" + quote(")")).r + val variableRegex = (quote("[") + "[^" + quote("{[()]}") + "]*" + quote("]")).r + val propertyRegex = (quote("${") + "[^" + quote("{[()]}") + "]*" + quote("}")).r sealed abstract class PatternPart(val effectiveStart: Int, val effectiveEnd: Int) extends Product with Serializable { require(effectiveStart <= effectiveEnd) @@ -67,19 +68,32 @@ object IvyRepository { } } + def substituteProperties(s: String, properties: Map[String, String]): String = + propertyRegex.findAllMatchIn(s).toVector.foldRight(s) { case (m, s0) => + val key = s0.substring(m.start + "${".length, m.end - "}".length) + val value = properties.getOrElse(key, "") + s0.take(m.start) + value + s0.drop(m.end) + } + } -case class IvyRepository(pattern: String, changing: Option[Boolean] = None) extends Repository { +case class IvyRepository( + pattern: String, + changing: Option[Boolean] = None, + properties: Map[String, String] = Map.empty +) extends Repository { import Repository._ import IvyRepository._ + private val pattern0 = substituteProperties(pattern, properties) + val parts = { - val optionalParts = optionalPartRegex.findAllMatchIn(pattern).toList.map { m => + val optionalParts = optionalPartRegex.findAllMatchIn(pattern0).toList.map { m => PatternPart.Optional(m.start, m.end) } - val len = pattern.length + val len = pattern0.length @tailrec def helper( @@ -105,16 +119,16 @@ case class IvyRepository(pattern: String, changing: Option[Boolean] = None) exte helper(0, optionalParts, Nil) } - assert(pattern.isEmpty == parts.isEmpty) - if (pattern.nonEmpty) { + assert(pattern0.isEmpty == parts.isEmpty) + if (pattern0.nonEmpty) { for ((a, b) <- parts.zip(parts.tail)) assert(a.end == b.start) assert(parts.head.start == 0) - assert(parts.last.end == pattern.length) + assert(parts.last.end == pattern0.length) } private val substituteHelpers = parts.map { part => - part(pattern.substring(part.effectiveStart, part.effectiveEnd)) + part(pattern0.substring(part.effectiveStart, part.effectiveEnd)) } def substitute(variables: Map[String, String]): String \/ String = @@ -154,7 +168,13 @@ case class IvyRepository(pattern: String, changing: Option[Boolean] = None) exte def artifacts(dependency: Dependency, project: Project) = project .publications - .collect { case (conf, p) if conf == "*" || conf == dependency.configuration => p } + .collect { + case (conf, p) + if conf == "*" || + conf == dependency.configuration || + project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) => + p + } .flatMap { p => substitute(variables( dependency.module.organization, diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala index 617c7f796..bc3d52ebb 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -66,7 +66,7 @@ object IvyXml { private def publications(node: Node): Map[String, Seq[Publication]] = node.children .filter(_.label == "artifact") - .flatMap { node0 => + .flatMap { node => val name = node.attribute("name").getOrElse("") val type0 = node.attribute("type").getOrElse("jar") val ext = node.attribute("ext").getOrElse(type0) @@ -116,12 +116,14 @@ object IvyXml { if (publicationsOpt.isEmpty) // no publications node -> default JAR artifact Seq("*" -> Publication(module.name, "jar", "jar")) - else + else { // publications node is there -> only its content (if it is empty, no artifacts, // as per the Ivy manual) + val inAllConfs = publicationsOpt.flatMap(_.get("*")).getOrElse(Nil) configurations0.flatMap { case (conf, _) => - publicationsOpt.flatMap(_.get(conf)).getOrElse(Nil).map(conf -> _) + (publicationsOpt.flatMap(_.get(conf)).getOrElse(Nil) ++ inAllConfs).map(conf -> _) } + } ) } \ No newline at end of file diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala new file mode 100644 index 000000000..c55f64ace --- /dev/null +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -0,0 +1,257 @@ +package coursier + +import java.io.{ File, OutputStreamWriter } + +import coursier.cli.TermDisplay +import sbt.{ MavenRepository => _, _ } +import sbt.Keys._ + +import scalaz.{ -\/, \/- } +import scalaz.concurrent.Task + +object CoursierPlugin extends AutoPlugin { + + override def trigger = allRequirements + + override def requires = sbt.plugins.IvyPlugin + + private def errPrintln(s: String): Unit = scala.Console.err.println(s) + + object autoImport { + val coursierParallelDownloads = Keys.coursierParallelDownloads + val coursierMaxIterations = Keys.coursierMaxIterations + val coursierChecksums = Keys.coursierChecksums + val coursierCachePolicy = Keys.coursierCachePolicy + val coursierResolvers = Keys.coursierResolvers + val coursierCache = Keys.coursierCache + val coursierProject = Keys.coursierProject + val coursierProjects = Keys.coursierProjects + } + + import autoImport._ + + + private val ivyProperties = Map( + "ivy.home" -> s"${sys.props("user.home")}/.ivy2" + ) ++ sys.props + + private def createLogger() = Some { + if (sys.env.get("COURSIER_NO_TERM").nonEmpty) + new coursier.Files.Logger { + override def downloadingArtifact(url: String, file: File): Unit = { + println(s"$url\n -> $file") + } + override def downloadedArtifact(url: String, success: Boolean): Unit = { + println(s"$url: ${if (success) "Success" else "Failed"}") + } + def init() = {} + } + else + new TermDisplay(new OutputStreamWriter(System.err)) + } + + + private def task = Def.task { + // let's update only one module at once, for a better output + // Downloads are already parallel, no need to parallelize further anyway + synchronized { + + val (currentProject, _) = coursierProject.value + val projects = coursierProjects.value + + val parallelDownloads = coursierParallelDownloads.value + val checksums = coursierChecksums.value + val maxIterations = coursierMaxIterations.value + val cachePolicy = coursierCachePolicy.value + val cacheDir = coursierCache.value + + val resolvers = coursierResolvers.value + + + val startRes = Resolution( + currentProject.dependencies.map { case (_, dep) => dep }.toSet, + filter = Some(dep => !dep.optional), + forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap + ) + + val interProjectRepo = InterProjectRepository(projects) + val repositories = interProjectRepo +: resolvers.flatMap(FromSbt.repository(_, ivyProperties)) + + val files = Files( + Seq("http://" -> new File(cacheDir, "http"), "https://" -> new File(cacheDir, "https")), + () => ???, + concurrentDownloadCount = parallelDownloads + ) + + val logger = createLogger() + logger.foreach(_.init()) + val fetch = coursier.Fetch( + repositories, + files.fetch(checksums = checksums, logger = logger)(cachePolicy = CachePolicy.LocalOnly), + files.fetch(checksums = checksums, logger = logger)(cachePolicy = cachePolicy) + ) + + def depsRepr = currentProject.dependencies.map { case (config, dep) => + s"${dep.module}:${dep.version}:$config->${dep.configuration}" + }.sorted + + errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") + for (depRepr <- depsRepr) + errPrintln(s" $depRepr") + + val res = startRes + .process + .run(fetch, maxIterations) + .attemptRun + .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) + .merge + + if (!res.isDone) + throw new Exception(s"Maximum number of iteration reached!") + + errPrintln("Resolution done") + + def repr(dep: Dependency) = { + // dep.version can be an interval, whereas the one from project can't + val version = res + .projectCache + .get(dep.moduleVersion) + .map(_._2.version) + .getOrElse(dep.version) + val extra = + if (version == dep.version) "" + else s" ($version for ${dep.version})" + + ( + Seq( + dep.module.organization, + dep.module.name, + dep.attributes.`type` + ) ++ + Some(dep.attributes.classifier) + .filter(_.nonEmpty) + .toSeq ++ + Seq( + version + ) + ).mkString(":") + extra + } + + if (res.conflicts.nonEmpty) { + // Needs test + println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") + } + + val errors = res.errors + if (errors.nonEmpty) { + println(s"\n${errors.size} error(s):") + for ((dep, errs) <- errors) { + println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") + } + } + + val trDepsWithArtifactsTasks = res.artifacts + .toVector + .map { a => + files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) + } + + errPrintln(s"Fetching artifacts") + // rename + val trDepsWithArtifacts = Task.gatherUnordered(trDepsWithArtifactsTasks).attemptRun match { + case -\/(ex) => + throw new Exception(s"Error while downloading / verifying artifacts", ex) + case \/-(l) => l.toMap + } + errPrintln(s"Fetching artifacts: done") + + val configs = ivyConfigurations.value.map(c => c.name -> c.extendsConfigs.map(_.name)).toMap + def allExtends(c: String) = { + // possibly bad complexity + def helper(current: Set[String]): Set[String] = { + val newSet = current ++ current.flatMap(configs.getOrElse(_, Nil)) + if ((newSet -- current).nonEmpty) + helper(newSet) + else + newSet + } + + helper(Set(c)) + } + + val depsByConfig = currentProject + .dependencies + .groupBy { case (c, _) => c } + .map { case (c, l) => + c -> l.map { case (_, d) => d } + } + + val sbtModuleReportsPerScope = configs.map { case (c, _) => c -> { + val a = allExtends(c).flatMap(depsByConfig.getOrElse(_, Nil)) + res.part(a) + .dependencyArtifacts + .groupBy { case (dep, _) => dep } + .map { case (dep, l) => dep -> l.map { case (_, a) => a } } + .map { case (dep, artifacts) => + val fe = artifacts.map { a => + a -> trDepsWithArtifacts.getOrElse(a, -\/("Not downloaded")) + } + new ModuleReport( + ModuleID(dep.module.organization, dep.module.name, dep.version, configurations = Some(dep.configuration)), + fe.collect { case (artifact, \/-(file)) => + if (file.toString.contains("file:/")) + throw new Exception(s"Wrong path: $file") + ToSbt.artifact(dep.module, artifact) -> file + }, + fe.collect { case (artifact, -\/(e)) => + errPrintln(s"${artifact.url}: $e") + ToSbt.artifact(dep.module, artifact) + }, + None, + None, + None, + None, + false, + None, + None, + None, + None, + Map.empty, + None, + None, + Nil, + Nil, + Nil + ) + } + }} + + new UpdateReport( + null, + sbtModuleReportsPerScope.toVector.map { case (c, r) => + new ConfigurationReport( + c, + r.toVector, + Nil, + Nil + ) + }, + new UpdateStats(-1L, -1L, -1L, cached = false), + Map.empty + ) + } + } + + override lazy val projectSettings = Seq( + coursierParallelDownloads := 6, + coursierMaxIterations := 50, + coursierChecksums := Seq(Some("SHA-1"), Some("MD5")), + coursierCachePolicy := CachePolicy.FetchMissing, + coursierResolvers <<= Tasks.coursierResolversTask, + coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), + update <<= task, + coursierProject <<= Tasks.coursierProjectTask, + coursierProjects <<= Tasks.coursierProjectsTask + ) + +} diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala new file mode 100644 index 000000000..b7ea73e44 --- /dev/null +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -0,0 +1,132 @@ +package coursier + +import coursier.ivy.IvyRepository +import sbt.{ Resolver, CrossVersion, ModuleID } + +object FromSbt { + + def sbtModuleIdName( + moduleId: ModuleID, + scalaVersion: => String, + scalaBinaryVersion: => String + ): String = moduleId.crossVersion match { + case CrossVersion.Disabled => moduleId.name + case f: CrossVersion.Full => moduleId.name + "_" + f.remapVersion(scalaVersion) + case f: CrossVersion.Binary => moduleId.name + "_" + f.remapVersion(scalaBinaryVersion) + } + + def mappings(mapping: String): Seq[(String, String)] = + mapping.split(';').flatMap { m => + val (froms, tos) = m.split("->", 2) match { + case Array(from) => (from, "default(compile)") + case Array(from, to) => (from, to) + } + + for { + from <- froms.split(',') + to <- tos.split(',') + } yield (from, to) + } + + def dependencies( + module: ModuleID, + scalaVersion: String, + scalaBinaryVersion: String + ): Seq[(String, Dependency)] = { + + // TODO Warn about unsupported properties in `module` + + val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) + + val dep = Dependency( + Module(module.organization, fullName), + module.revision, + exclusions = module.exclusions.map { rule => + // FIXME Other `rule` fields are ignored here + (rule.organization, rule.name) + }.toSet + ) + + val mapping = module.configurations.getOrElse("compile") + val allMappings = mappings(mapping) + + val attributes = + if (module.explicitArtifacts.isEmpty) + Seq(Attributes()) + else + module.explicitArtifacts.map { a => + Attributes(`type` = a.extension, classifier = a.classifier.getOrElse("")) + } + + for { + (from, to) <- allMappings.toSeq + attr <- attributes + } yield from -> dep.copy(configuration = to, attributes = attr) + } + + def project( + projectID: ModuleID, + allDependencies: Seq[ModuleID], + ivyConfigurations: Map[String, Seq[String]], + scalaVersion: String, + scalaBinaryVersion: String + ): Project = { + + // FIXME Ignored for now + // val sbtDepOverrides = dependencyOverrides.value + // val sbtExclusions = excludeDependencies.value + + val deps = allDependencies.flatMap(dependencies(_, scalaVersion, scalaBinaryVersion)) + + Project( + Module(projectID.organization, sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion)), + projectID.revision, + deps, + ivyConfigurations, + None, + Nil, + Map.empty, + Nil, + None, + None, + Nil + ) + } + + def repository(resolver: Resolver, ivyProperties: Map[String, String]): Option[Repository] = + resolver match { + case sbt.MavenRepository(_, root) => + if (root.startsWith("http://") || root.startsWith("https://")) { + val root0 = if (root.endsWith("/")) root else root + "/" + Some(MavenRepository(root0)) + } else { + Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it") + None + } + + case sbt.FileRepository(_, _, patterns) + if patterns.ivyPatterns.lengthCompare(1) == 0 && + patterns.ivyPatterns == patterns.artifactPatterns => + + Some(IvyRepository( + "file://" + patterns.ivyPatterns.head, + changing = Some(true), + properties = ivyProperties + )) + + case sbt.URLRepository(_, patterns) + if patterns.ivyPatterns.lengthCompare(1) == 0 && + patterns.ivyPatterns == patterns.artifactPatterns => + + Some(IvyRepository( + patterns.ivyPatterns.head, + changing = None, + properties = ivyProperties + )) + + case other => + Console.err.println(s"Warning: unrecognized repository ${other.name}, ignoring it") + None + } + +} diff --git a/plugin/src/main/scala/coursier/InterProjectRepository.scala b/plugin/src/main/scala/coursier/InterProjectRepository.scala new file mode 100644 index 000000000..18233c8fe --- /dev/null +++ b/plugin/src/main/scala/coursier/InterProjectRepository.scala @@ -0,0 +1,50 @@ +package coursier + +import scalaz.{ -\/, \/-, Monad, EitherT } + +case class InterProjectSource(artifacts: Map[(Module, String), Map[String, Seq[Artifact]]]) extends Artifact.Source { + def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = + artifacts + .get(dependency.moduleVersion) + .toSeq + .flatMap(_.get(dependency.configuration)) + .flatten +} + +case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artifact])])]) extends Repository { + + Console.err.println("InterProjectRepository") + for ((p, _) <- projects) + Console.err.println(s" ${p.module}:${p.version}") + + private val map = projects + .map { case (proj, a) => proj.moduleVersion -> proj } + .toMap + + val source = InterProjectSource( + projects.map { case (proj, a) => + val artifacts = a.toMap + val allArtifacts = proj.allConfigurations.map { case (c, ext) => + c -> ext.toSeq.flatMap(artifacts.getOrElse(_, Nil)) + } + proj.moduleVersion -> allArtifacts + }.toMap + ) + + def find[F[_]]( + module: Module, + version: String, + fetch: Fetch.Content[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, (Artifact.Source, Project)] = { + val res = map.get((module, version)) match { + case Some(proj) => + \/-((source, proj)) + case None => + -\/(s"Project not found: $module:$version") + } + + EitherT(F.point(res)) + } +} \ No newline at end of file diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala new file mode 100644 index 000000000..cf6c56990 --- /dev/null +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -0,0 +1,18 @@ +package coursier + +import java.io.File +import sbt.{ Resolver, SettingKey, TaskKey } + +object Keys { + val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads", "") // 6 + val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations", "") // 50 + val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "") //Seq(Some("SHA-1"), Some("MD5")) + val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "") // = CachePolicy.FetchMissing + + val coursierResolvers = TaskKey[Seq[Resolver]]("coursier-resolvers", "") + + val coursierCache = SettingKey[File]("coursier-cache", "") + + val coursierProject = TaskKey[(Project, Seq[(String, Seq[Artifact])])]("coursier-project", "") + val coursierProjects = TaskKey[Seq[(Project, Seq[(String, Seq[Artifact])])]]("coursier-projects", "") +} diff --git a/plugin/src/main/scala/coursier/Structure.scala b/plugin/src/main/scala/coursier/Structure.scala new file mode 100644 index 000000000..c9e5ea7e3 --- /dev/null +++ b/plugin/src/main/scala/coursier/Structure.scala @@ -0,0 +1,40 @@ +package coursier + +import sbt._ + +// things from sbt-structure +object Structure { + import Def.Initialize._ + + def structure(state: State): Load.BuildStructure = + sbt.Project.structure(state) + + implicit def `enrich SettingKey`[T](key: SettingKey[T]) = new { + def find(state: State): Option[T] = + key.get(structure(state).data) + + def get(state: State): T = + find(state).get + + def getOrElse(state: State, default: => T): T = + find(state).getOrElse(default) + } + + implicit def `enrich TaskKey`[T](key: TaskKey[T]) = new { + def find(state: State): Option[sbt.Task[T]] = + key.get(structure(state).data) + + def get(state: State): sbt.Task[T] = + find(state).get + + def forAllProjects(state: State, projects: Seq[ProjectRef]): sbt.Task[Map[ProjectRef, T]] = { + val tasks = projects.flatMap(p => key.in(p).get(structure(state).data).map(_.map(it => (p, it)))) + std.TaskExtra.joinTasks(tasks).join.map(_.toMap) + } + + def forAllConfigurations(state: State, configurations: Seq[sbt.Configuration]): sbt.Task[Map[sbt.Configuration, T]] = { + val tasks = configurations.flatMap(c => key.in(c).get(structure(state).data).map(_.map(it => (c, it)))) + std.TaskExtra.joinTasks(tasks).join.map(_.toMap) + } + } +} diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala new file mode 100644 index 000000000..130af086b --- /dev/null +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -0,0 +1,69 @@ +package coursier + +import sbt.{Classpaths, Resolver, Def} +import Structure._ +import Keys._ +import sbt.Keys._ + +object Tasks { + + def coursierResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = Def.task { + var l = externalResolvers.value + if (sbtPlugin.value) + l = Seq( + sbtResolver.value, + Classpaths.sbtPluginReleases + ) ++ l + l + } + + def coursierProjectTask: Def.Initialize[sbt.Task[(Project, Seq[(String, Seq[Artifact])])]] = + ( + sbt.Keys.state, + sbt.Keys.thisProjectRef + ).flatMap { (state, projectRef) => + + // should projectID.configurations be used instead? + val configurations = ivyConfigurations.in(projectRef).get(state) + + // exportedProducts looks like what we want, but depends on the update task, which + // make the whole thing run into cycles... + val artifacts = configurations.map { cfg => + cfg.name -> Option(classDirectory.in(projectRef).in(cfg).getOrElse(state, null)) + }.collect { case (name, Some(classDir)) => + name -> Seq( + Artifact( + classDir.toURI.toString, + Map.empty, + Map.empty, + Attributes(), + changing = true + ) + ) + } + + val allDependenciesTask = allDependencies.in(projectRef).get(state) + + for { + allDependencies <- allDependenciesTask + } yield { + + val proj = FromSbt.project( + projectID.in(projectRef).get(state), + allDependencies, + configurations.map { cfg => cfg.name -> cfg.extendsConfigs.map(_.name) }.toMap, + scalaVersion.in(projectRef).get(state), + scalaBinaryVersion.in(projectRef).get(state) + ) + + (proj, artifacts) + } + } + + def coursierProjectsTask: Def.Initialize[sbt.Task[Seq[(Project, Seq[(String, Seq[Artifact])])]]] = + sbt.Keys.state.flatMap { state => + val projects = structure(state).allProjectRefs + coursierProject.forAllProjects(state, projects).map(_.values.toVector) + } + +} diff --git a/plugin/src/main/scala/coursier/ToSbt.scala b/plugin/src/main/scala/coursier/ToSbt.scala new file mode 100644 index 000000000..6a83eaa78 --- /dev/null +++ b/plugin/src/main/scala/coursier/ToSbt.scala @@ -0,0 +1,18 @@ +package coursier + +import sbt._ + +object ToSbt { + + def artifact(module: Module, artifact: Artifact): sbt.Artifact = + sbt.Artifact( + s"${module.organization}:${module.name}", + artifact.attributes.`type`, + "jar", + Some(artifact.attributes.classifier), + Nil, + Some(url(artifact.url)), + Map.empty + ) + +} From 799179e74d06aebe502334c612be7b89966d6bcb Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:34 +0100 Subject: [PATCH 10/80] Add tut support --- build.sbt | 12 ++- doc/README.md | 187 ++++++++++++++++++++++++++++++++++++++++++++ project/plugins.sbt | 1 + 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 doc/README.md diff --git a/build.sbt b/build.sbt index 5cdc80b07..4709eb57b 100644 --- a/build.sbt +++ b/build.sbt @@ -204,6 +204,16 @@ lazy val web = project ) ) +lazy val doc = project + .dependsOn(coreJvm, files) + .settings(commonSettings) + .settings(noPublishSettings) + .settings(tutSettings) + .settings( + tutSourceDirectory := baseDirectory.value, + tutTargetDirectory := baseDirectory.value / ".." + ) + // Don't try to compile that if you're not in 2.10 lazy val plugin = project .dependsOn(coreJvm, files, cli) @@ -214,7 +224,7 @@ lazy val plugin = project ) lazy val `coursier` = project.in(file(".")) - .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web) + .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web, doc) .settings(commonSettings) .settings(noPublishSettings) .settings(releaseSettings) diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 000000000..16c749903 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,187 @@ +# Coursier + +*Pure Scala Artifact Fetching* + +A pure Scala substitute for [Aether](http://www.eclipse.org/aether/) + +[![Build Status](https://travis-ci.org/alexarchambault/coursier.svg?branch=master)](https://travis-ci.org/alexarchambault/coursier) +[![Build status (Windows)](https://ci.appveyor.com/api/projects/status/trtum5b7washfbj9?svg=true)](https://ci.appveyor.com/project/alexarchambault/coursier) +[![Join the chat at https://gitter.im/alexarchambault/coursier](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/alexarchambault/coursier?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Maven Central](https://img.shields.io/maven-central/v/com.github.alexarchambault/coursier_2.11.svg)](https://maven-badges.herokuapp.com/maven-central/com.github.alexarchambault/coursier_2.11) + +*coursier* is a dependency resolver / fetcher *à la* Maven / Ivy, entirely +rewritten from scratch in Scala. It aims at being fast and easy to embed +in other contexts. Its very core (`core` module) aims at being +extremely pure, and should be approached thinking algebraically. + +The `files` module handles caching of the metadata and artifacts themselves, +and is less so pure than the `core` module, in the sense that it happily +does IO as a side-effect (although it naturally favors immutability for all +that's kept in memory). + +It handles fancy Maven features like +* [POM inheritance](http://books.sonatype.com/mvnref-book/reference/pom-relationships-sect-project-relationships.html#pom-relationships-sect-project-inheritance), +* [dependency management](http://books.sonatype.com/mvnex-book/reference/optimizing-sect-dependencies.html), +* [import scope](https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Importing_Dependencies), +* [properties](http://books.sonatype.com/mvnref-book/reference/resource-filtering-sect-properties.html), +* etc. + +It happily resolves dependencies involving modules from the Hadoop ecosystem (Spark, Flink, etc.), that +make a heavy use of these. + +It can be used either from the command-line, via its API, or from the browser. + +It downloads the metadata or the artifacts in parallel (usually, 6 parallel +downloads). + +## Command-line + +Download and run its laucher with +``` +$ curl -L -o coursier https://git.io/vBSmI && chmod +x coursier && ./coursier --help +``` + +Note that the launcher itself weights only 8 kB and can be easily +embedded as is in other projects. +The first time it is run, it will download the artifacts required to launch +coursier. You'll be fine the next times :-). + +The cache of this default launcher defaults to a directory named `.coursier`, +in the same directory as the launcher. This can be changed by manually adjusting +the `COURSIER_CACHE` variable in the first lines of the launcher. + +``` +$ ./coursier --help +``` +lists the available coursier commands. The most notable ones are `launch`, +`fetch`, and `classpath`. Type +``` +$ ./coursier command --help +``` +to get a description of the various options the command `command` (replace with one +of the above command) accepts. + +### launch + +The `launch` command fetches a set of Maven coordinates it is given, along +with their transitive dependencies, then launches the "main `main` class" from +it if it can find one (typically from the manifest of the first coordinates). +The main class to launch can also be manually specified with the `-M` option. + +For example, it can launch: + +* [Ammonite](https://github.com/lihaoyi/Ammonite) (enhanced Scala REPL), +``` +$ ./coursier launch com.lihaoyi:ammonite-repl_2.11.7:0.5.0 +``` + +along with the REPLs of various JVM languages like + +* Frege, +``` +$ ./coursier launch -r central -r https://oss.sonatype.org/content/groups/public \ + org.frege-lang:frege-repl-core:1.3 -M frege.repl.FregeRepl +``` + +* clojure, +``` +$ ./coursier launch org.clojure:clojure:1.7.0 -M clojure.main +``` + +* jruby, +``` +$ wget https://raw.githubusercontent.com/jruby/jruby/master/bin/jirb && \ + ./coursier launch org.jruby:jruby:9.0.4.0 -M org.jruby.Main -- -- jirb +``` + +* jython, +``` +$ ./coursier launch org.python:jython-standalone:2.7.0 -M org.python.util.jython +``` + +* Groovy, +``` +$ ./coursier launch org.codehaus.groovy:groovy-groovysh:2.4.5 -M org.codehaus.groovy.tools.shell.Main \ + commons-cli:commons-cli:1.3.1 +``` + +etc. + +and various programs, like + +* Proguard and its utility Retrace, +``` +$ ./coursier launch net.sf.proguard:proguard-base:5.2.1 -M proguard.ProGuard +$ ./coursier launch net.sf.proguard:proguard-retrace:5.2.1 -M proguard.retrace.ReTrace +``` + +### fetch + +The `fetch` command simply fetches a set of dependencies, along with their +transitive dependencies, then prints the local paths of all their artefacts. + +Example +``` +$ ./coursier fetch org.apache.spark:spark-sql_2.11:1.5.2 +... +/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/io/dropwizard/metrics/metrics-jvm/3.1.2/metrics-jvm-3.1.2.jar +/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/javax/servlet/javax.servlet-api/3.0.1/javax.servlet-api-3.0.1.jar +/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/javax/inject/javax.inject/1/javax.inject-1.jar +... +``` + +### classpath + +The `classpath` command transitively fetches a set of dependencies like +`fetch` does, then prints a classpath that can be handed over directly +to `java`, like +``` +$ java -cp "$(./coursier classpath com.lihaoyi:ammonite-repl_2.11.7:0.5.0 | tail -n1)" ammonite.repl.Repl +Loading... +Welcome to the Ammonite Repl 0.5.0 +(Scala 2.11.7 Java 1.8.0_60) +@ +``` + +## API + +This [gist](https://gist.github.com/larsrh/42da43aa74dc4e78aa59) by [Lars Hupel](https://github.com/larsrh/) +illustrates how the API of coursier can be used to get transitives dependencies +and fetch the corresponding artefacts. + +More explanations to come :-) + +## Scala JS demo + +*coursier* is also compiled to Scala JS, and can be tested in the browser via its +[demo](http://alexarchambault.github.io/coursier/#demo). + +# To do / missing + +- Snapshots metadata / artifacts, once in cache, are not automatically +updated for now. [#41](https://github.com/alexarchambault/coursier/issues/41) +- File locking could be better (none for metadata, no re-attempt if file locked elsewhere for artifacts) [#71](https://github.com/alexarchambault/coursier/issues/71) +- Handle "configurations" like Ivy does, instead of just the standard +(hard-coded) Maven "scopes" [#8](https://github.com/alexarchambault/coursier/issues/8) +- SBT plugin [#52](https://github.com/alexarchambault/coursier/issues/52), +requires Ivy-like configurations [#8](https://github.com/alexarchambault/coursier/issues/8) + +See the list of [issues](https://github.com/alexarchambault/coursier/issues). + +# Contributors + +- Your name here :-) + +Don't hesitate to pick an issue to contribute, and / or ask for help for how to proceed +on the [Gitter channel](https://gitter.im/alexarchambault/coursier). + +# Projects using coursier + +- [Lars Hupel](https://github.com/larsrh/)'s [libisabelle](https://github.com/larsrh/libisabelle) fetches +some of its requirements via coursier, +- [jupyter-scala](https://github.com/alexarchambault/jupyter-scala) should soon allow +to add dependencies in its sessions with coursier (initial motivation for writing coursier), +- Your project here :-) + + +Released under the Apache license, v2. diff --git a/project/plugins.sbt b/project/plugins.sbt index d26d5c6bc..ad7b57ef9 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,3 +3,4 @@ addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.5") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.1.0") +addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.4.0") From 22f8626e61d35cbe9b5d8af03ccb4b0a401f2eeb Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:34 +0100 Subject: [PATCH 11/80] Fix in build.sbt --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 4709eb57b..353a10f92 100644 --- a/build.sbt +++ b/build.sbt @@ -58,7 +58,8 @@ lazy val baseCommonSettings = Seq( javacOptions ++= Seq( "-source", "1.7", "-target", "1.7" - ) + ), + javacOptions in Keys.doc := Seq() ) lazy val commonSettings = baseCommonSettings ++ Seq( @@ -155,8 +156,7 @@ lazy val bootstrap = project artifactName0(sv, m, artifact) }, crossPaths := false, - autoScalaLibrary := false, - javacOptions in doc := Seq() + autoScalaLibrary := false ) lazy val cli = project From 6e2905d9059233841421edbadba0263f42eb9ca5 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:34 +0100 Subject: [PATCH 12/80] Fix in TermDisplay --- cli/src/main/scala/coursier/cli/TermDisplay.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cli/src/main/scala/coursier/cli/TermDisplay.scala index dee09ea6c..78e00fd42 100644 --- a/cli/src/main/scala/coursier/cli/TermDisplay.scala +++ b/cli/src/main/scala/coursier/cli/TermDisplay.scala @@ -70,6 +70,16 @@ class TermDisplay(out: Writer) extends Logger { out.write(s"$url0 $extra0\n") } + if (downloads0.length < lineCount) { + for (_ <- downloads0.length until lineCount) { + ansi.clearLine(2) + ansi.down(1) + } + + for (_ <- downloads0.length until lineCount) + ansi.up(1) + } + for (_ <- downloads0.indices) ansi.up(1) From ca17da83f1d23871dd87fc4375f82753ad064f3d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:35 +0100 Subject: [PATCH 13/80] Fallback output --- .../main/scala/coursier/cli/TermDisplay.scala | 132 +++++++++++++----- .../main/scala/coursier/CoursierPlugin.scala | 16 +-- 2 files changed, 103 insertions(+), 45 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cli/src/main/scala/coursier/cli/TermDisplay.scala index 78e00fd42..e4c7358b9 100644 --- a/cli/src/main/scala/coursier/cli/TermDisplay.scala +++ b/cli/src/main/scala/coursier/cli/TermDisplay.scala @@ -10,15 +10,58 @@ import coursier.Files.Logger import scala.annotation.tailrec import scala.collection.mutable.ArrayBuffer -class TermDisplay(out: Writer) extends Logger { +class TermDisplay( + out: Writer, + var fallbackMode: Boolean = false +) extends Logger { private val ansi = new Ansi(out) private var width = 80 private val refreshInterval = 1000 / 60 + private val fallbackRefreshInterval = 1000 + private val lock = new AnyRef private val t = new Thread("TermDisplay") { override def run() = lock.synchronized { + val baseExtraWidth = width / 5 + + def reflowed(url: String, info: Info) = { + val pctOpt = info.pct.map(100.0 * _) + val extra = + if (info.length.isEmpty && info.downloaded == 0L) + "" + else + s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${info.downloaded}${info.length.map(" / " + _).mkString})" + + val total = url.length + 1 + extra.length + val (url0, extra0) = + if (total >= width) { // or > ? If equal, does it go down 2 lines? + val overflow = total - width + 1 + + val extra0 = + if (extra.length > baseExtraWidth) + extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…" + else + extra + + val total0 = url.length + 1 + extra0.length + val overflow0 = total0 - width + 1 + + val url0 = + if (total0 >= width) + url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…" + else + url + + (url0, extra0) + } else + (url, extra) + + (url0, extra0) + } + + @tailrec def helper(lineCount: Int): Unit = Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { case None => helper(lineCount) @@ -35,36 +78,8 @@ class TermDisplay(out: Writer) extends Logger { for ((url, info) <- downloads0) { assert(info != null, s"Incoherent state ($url)") - val pctOpt = info.pct.map(100.0 * _) - val extra = - if (info.length.isEmpty && info.downloaded == 0L) - "" - else - s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${info.downloaded}${info.length.map(" / " + _).mkString})" - val total = url.length + 1 + extra.length - val (url0, extra0) = - if (total >= width) { // or > ? If equal, does it go down 2 lines? - val overflow = total - width + 1 - - val extra0 = - if (extra.length > baseExtraWidth) - extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…" - else - extra - - val total0 = url.length + 1 + extra0.length - val overflow0 = total0 - width + 1 - - val url0 = - if (total0 >= width) - url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…" - else - url - - (url0, extra0) - } else - (url, extra) + val (url0, extra0) = reflowed(url, info) ansi.clearLine(2) out.write(s"$url0 $extra0\n") @@ -88,15 +103,54 @@ class TermDisplay(out: Writer) extends Logger { helper(downloads0.length) } - helper(0) + + @tailrec def fallbackHelper(previous: Set[String]): Unit = + Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { + case None => fallbackHelper(previous) + case Some(Left(())) => // poison pill + case Some(Right(())) => + val downloads0 = downloads.synchronized { + downloads + .toVector + .map { url => url -> infos.get(url) } + .sortBy { case (_, info) => - info.pct.sum } + } + + var displayedSomething = false + for ((url, info) <- downloads0 if previous(url)) { + assert(info != null, s"Incoherent state ($url)") + + val (url0, extra0) = reflowed(url, info) + + displayedSomething = true + out.write(s"$url0 $extra0\n") + } + + if (displayedSomething) + out.write("\n") + + out.flush() + Thread.sleep(fallbackRefreshInterval) + fallbackHelper(previous ++ downloads0.map { case (url, _) => url }) + } + + if (fallbackMode) + fallbackHelper(Set.empty) + else + helper(0) } } t.setDaemon(true) def init(): Unit = { - width = TTY.consoleDim("cols") - ansi.clearLine(2) + try { + width = TTY.consoleDim("cols") + ansi.clearLine(2) + } catch { case _: Exception => + fallbackMode = true + } + t.start() } @@ -123,6 +177,12 @@ class TermDisplay(out: Writer) extends Logger { val prev = infos.putIfAbsent(url, Info(0L, None)) assert(prev == null) + if (fallbackMode) { + // FIXME What about concurrent accesses to out from the thread above? + out.write(s"Downloading $url\n") + out.flush() + } + downloads.synchronized { downloads.append(url) } @@ -150,6 +210,12 @@ class TermDisplay(out: Writer) extends Logger { downloads -= url } + if (fallbackMode) { + // FIXME What about concurrent accesses to out from the thread above? + out.write(s"Downloaded $url\n") + out.flush() + } + val info = infos.remove(url) assert(info != null) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index c55f64ace..182de2b7d 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -36,18 +36,10 @@ object CoursierPlugin extends AutoPlugin { ) ++ sys.props private def createLogger() = Some { - if (sys.env.get("COURSIER_NO_TERM").nonEmpty) - new coursier.Files.Logger { - override def downloadingArtifact(url: String, file: File): Unit = { - println(s"$url\n -> $file") - } - override def downloadedArtifact(url: String, success: Boolean): Unit = { - println(s"$url: ${if (success) "Success" else "Failed"}") - } - def init() = {} - } - else - new TermDisplay(new OutputStreamWriter(System.err)) + new TermDisplay( + new OutputStreamWriter(System.err), + fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty + ) } From eaa78748740e02d706662b6ec014e57f6470c3db Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:35 +0100 Subject: [PATCH 14/80] Move Maven specific method closer to its usage --- .../main/scala/coursier/core/Repository.scala | 23 ---------------- .../scala/coursier/maven/MavenSource.scala | 26 +++++++++++++++++++ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Repository.scala b/core/shared/src/main/scala/coursier/core/Repository.scala index 2d0f96dcf..297f95f9a 100644 --- a/core/shared/src/main/scala/coursier/core/Repository.scala +++ b/core/shared/src/main/scala/coursier/core/Repository.scala @@ -38,29 +38,6 @@ object Repository { ) .withDefaultChecksums )) - def withJavadocSources: Artifact = { - val base = underlying.url.stripSuffix(".jar") - underlying.copy(extra = underlying.extra ++ Seq( - "sources" -> Artifact( - base + "-sources.jar", - Map.empty, - Map.empty, - Attributes("jar", "src"), // Are these the right attributes? - changing = underlying.changing - ) - .withDefaultChecksums - .withDefaultSignature, - "javadoc" -> Artifact( - base + "-javadoc.jar", - Map.empty, - Map.empty, - Attributes("jar", "javadoc"), // Same comment as above - changing = underlying.changing - ) - .withDefaultChecksums - .withDefaultSignature - )) - } } } diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 26973836d..06921e82d 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -11,6 +11,32 @@ case class MavenSource( import Repository._ import MavenRepository._ + private implicit class DocSourcesArtifactExtensions(val underlying: Artifact) { + def withJavadocSources: Artifact = { + val base = underlying.url.stripSuffix(".jar") + underlying.copy(extra = underlying.extra ++ Seq( + "sources" -> Artifact( + base + "-sources.jar", + Map.empty, + Map.empty, + Attributes("jar", "src"), // Are these the right attributes? + changing = underlying.changing + ) + .withDefaultChecksums + .withDefaultSignature, + "javadoc" -> Artifact( + base + "-javadoc.jar", + Map.empty, + Map.empty, + Attributes("jar", "javadoc"), // Same comment as above + changing = underlying.changing + ) + .withDefaultChecksums + .withDefaultSignature + )) + } + } + def artifacts( dependency: Dependency, project: Project From 0e5118befee3b3af0e1325a38e8cf12029e9ab94 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:35 +0100 Subject: [PATCH 15/80] Better handling of classifiers --- .../scala/coursier/core/Definitions.scala | 18 ++- .../main/scala/coursier/core/Resolution.scala | 12 +- .../scala/coursier/ivy/IvyRepository.scala | 76 ++++++---- .../src/main/scala/coursier/ivy/IvyXml.scala | 5 +- .../coursier/maven/MavenRepository.scala | 5 +- .../scala/coursier/maven/MavenSource.scala | 134 ++++++++---------- .../coursier/InterProjectRepository.scala | 21 ++- .../scala/coursier/test/TestRepository.scala | 6 +- 8 files changed, 157 insertions(+), 120 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 8576d835c..c741d6a2d 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -47,7 +47,10 @@ case class Dependency( case class Attributes( `type`: String, classifier: String -) +) { + def publication(name: String, ext: String): Publication = + Publication(name, `type`, ext, classifier) +} case class Project( module: Module, @@ -133,8 +136,11 @@ case class SnapshotVersioning( case class Publication( name: String, `type`: String, - ext: String -) + ext: String, + classifier: String +) { + def attributes: Attributes = Attributes(`type`, classifier) +} case class Artifact( url: String, @@ -146,6 +152,10 @@ case class Artifact( object Artifact { trait Source { - def artifacts(dependency: Dependency, project: Project): Seq[Artifact] + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ): Seq[Artifact] } } diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index f2e240bfd..4b9d16e15 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -746,16 +746,22 @@ case class Resolution( .getOrElse(Map.empty) ) - def artifacts: Seq[Artifact] = + private def artifacts0(overrideClassifiers: Option[Seq[String]]): Seq[Artifact] = for { dep <- minDependencies.toSeq (source, proj) <- projectCache .get(dep.moduleVersion) .toSeq artifact <- source - .artifacts(dep, proj) + .artifacts(dep, proj, overrideClassifiers) } yield artifact + def classifiersArtifacts(classifiers: Seq[String]): Seq[Artifact] = + artifacts0(Some(classifiers)) + + def artifacts: Seq[Artifact] = + artifacts0(None) + def dependencyArtifacts: Seq[(Dependency, Artifact)] = for { dep <- minDependencies.toSeq @@ -763,7 +769,7 @@ case class Resolution( .get(dep.moduleVersion) .toSeq artifact <- source - .artifacts(dep, proj) + .artifacts(dep, proj, None) } yield dep -> artifact def errors: Seq[(Dependency, Seq[String])] = diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index ac24d534a..b1ce1a788 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -165,37 +165,53 @@ case class IvyRepository( val source: Artifact.Source = new Artifact.Source { - def artifacts(dependency: Dependency, project: Project) = - project - .publications - .collect { - case (conf, p) - if conf == "*" || - conf == dependency.configuration || - project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) => - p - } - .flatMap { p => - substitute(variables( - dependency.module.organization, - dependency.module.name, - dependency.version, - p.`type`, - p.name, - p.ext - )).toList.map(p -> _) - } - .map { case (p, url) => - Artifact( - url, - Map.empty, - Map.empty, - Attributes(p.`type`, p.ext), - changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable - ) - .withDefaultChecksums - .withDefaultSignature + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ) = { + + val retained = + overrideClassifiers match { + case None => + project.publications.collect { + case (conf, p) + if conf == "*" || + conf == dependency.configuration || + project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) => + p + } + case Some(classifiers) => + val classifiersSet = classifiers.toSet + project.publications.collect { + case (_, p) if classifiersSet(p.classifier) => + p + } } + + val retainedWithUrl = retained.flatMap { p => + substitute(variables( + dependency.module.organization, + dependency.module.name, + dependency.version, + p.`type`, + p.name, + p.ext + )).toList.map(p -> _) + } + + retainedWithUrl.map { case (p, url) => + Artifact( + url, + Map.empty, + Map.empty, + p.attributes, + changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable + ) + .withDefaultChecksums + .withDefaultSignature + } + } } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala index bc3d52ebb..62b70cb8d 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -71,7 +71,8 @@ object IvyXml { val type0 = node.attribute("type").getOrElse("jar") val ext = node.attribute("ext").getOrElse(type0) val confs = node.attribute("conf").toOption.fold(Seq("*"))(_.split(',')) - confs.map(_ -> Publication(name, type0, ext)) + val classifier = node.attribute("classifier").toOption.getOrElse("") + confs.map(_ -> Publication(name, type0, ext, classifier)) } .groupBy { case (conf, _) => conf } .map { case (conf, l) => conf -> l.map { case (_, p) => p } } @@ -115,7 +116,7 @@ object IvyXml { None, if (publicationsOpt.isEmpty) // no publications node -> default JAR artifact - Seq("*" -> Publication(module.name, "jar", "jar")) + Seq("*" -> Publication(module.name, "jar", "jar", "")) else { // publications node is there -> only its content (if it is empty, no artifacts, // as per the Ivy manual) diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index d0522f13e..d05fcf472 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -244,7 +244,10 @@ case class MavenRepository( xml <- \/.fromEither(compatibility.xmlParse(str)) _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") proj <- Pom.project(xml) - } yield proj.copy(configurations = defaultConfigurations)): (String \/ Project) + } yield proj.copy( + configurations = defaultConfigurations, + publications = ??? + )): (String \/ Project) } } } diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 06921e82d..467c8584a 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -39,86 +39,74 @@ case class MavenSource( def artifacts( dependency: Dependency, - project: Project + project: Project, + overrideClassifiers: Option[Seq[String]] ): Seq[Artifact] = { - def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = - ivyLikePath( - dependency.module.organization, - dependency.module.name, - project.version, - subDir, - baseSuffix, - ext - ) - - val path = - if (ivyLike) - ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) - else { - val versioning = - project - .snapshotVersioning - .flatMap(versioning => - mavenVersioning(versioning, dependency.attributes.classifier, dependency.attributes.`type`) - ) - - dependency.module.organization.split('.').toSeq ++ Seq( - dependency.module.name, + def artifactOf(module: Module, publication: Publication) = { + def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = + ivyLikePath( + module.organization, + module.name, project.version, - s"${dependency.module.name}-${versioning getOrElse project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + subDir, + baseSuffix, + ext ) + + val path = + if (ivyLike) + ivyLikePath0(publication.`type` + "s", "", publication.ext) + else { + val versioning = + project + .snapshotVersioning + .flatMap(versioning => + mavenVersioning(versioning, publication.classifier, publication.`type`) + ) + + module.organization.split('.').toSeq ++ Seq( + module.name, + project.version, + s"${module.name}-${versioning getOrElse project.version}${Some(publication.classifier).filter(_.nonEmpty).map("-" + _).mkString}.${publication.ext}" + ) + } + + val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT")) + var artifact = + Artifact( + root + path.mkString("/"), + Map.empty, + Map.empty, + publication.attributes, + changing = changing0 + ) + .withDefaultChecksums + + if (publication.ext == "jar") { + artifact = artifact.withDefaultSignature } - val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT")) - var artifact = - Artifact( - root + path.mkString("/"), - Map.empty, - Map.empty, - dependency.attributes, - changing = changing0 - ) - .withDefaultChecksums - - if (dependency.attributes.`type` == "jar") { - artifact = artifact.withDefaultSignature - - // FIXME Snapshot versioning of sources and javadoc is not taken into account here. - // Will be ok if it's the same as the main JAR though. - - artifact = - if (ivyLike) { - val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") - val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") - - artifact - .copy( - extra = artifact.extra ++ Map( - "sources" -> Artifact( - srcPath, - Map.empty, - Map.empty, - Attributes("jar", "src"), // Are these the right attributes? - changing = changing0 - ) - .withDefaultChecksums - .withDefaultSignature, - "javadoc" -> Artifact( - javadocPath, - Map.empty, - Map.empty, - Attributes("jar", "javadoc"), // Same comment as above - changing = changing0 - ) - .withDefaultChecksums - .withDefaultSignature - )) - } else - artifact - .withJavadocSources + artifact } - Seq(artifact) + overrideClassifiers match { + case Some(classifiers) => + val classifiersSet = classifiers.toSet + project.publications.collect { + case (_, p) if classifiersSet(p.classifier) => + artifactOf(dependency.module, p) + } + case None => + Seq( + artifactOf( + dependency.module, + dependency.attributes.publication( + dependency.module.name, + dependency.attributes.`type` + ) + ) + ) + } } } diff --git a/plugin/src/main/scala/coursier/InterProjectRepository.scala b/plugin/src/main/scala/coursier/InterProjectRepository.scala index 18233c8fe..62b086989 100644 --- a/plugin/src/main/scala/coursier/InterProjectRepository.scala +++ b/plugin/src/main/scala/coursier/InterProjectRepository.scala @@ -3,12 +3,21 @@ package coursier import scalaz.{ -\/, \/-, Monad, EitherT } case class InterProjectSource(artifacts: Map[(Module, String), Map[String, Seq[Artifact]]]) extends Artifact.Source { - def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = - artifacts - .get(dependency.moduleVersion) - .toSeq - .flatMap(_.get(dependency.configuration)) - .flatten + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ): Seq[Artifact] = + overrideClassifiers match { + case None => + artifacts + .get(dependency.moduleVersion) + .toSeq + .flatMap(_.get(dependency.configuration)) + .flatten + case Some(_) => + Nil + } } case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artifact])])]) extends Repository { diff --git a/tests/shared/src/test/scala/coursier/test/TestRepository.scala b/tests/shared/src/test/scala/coursier/test/TestRepository.scala index fcbe0030f..bc851b8e1 100644 --- a/tests/shared/src/test/scala/coursier/test/TestRepository.scala +++ b/tests/shared/src/test/scala/coursier/test/TestRepository.scala @@ -8,7 +8,11 @@ import scalaz.Scalaz._ class TestRepository(projects: Map[(Module, String), Project]) extends Repository { val source = new core.Artifact.Source { - def artifacts(dependency: Dependency, project: Project) = ??? + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ) = ??? } def find[F[_]]( module: Module, From fab18dd99b52f5c71df1c68ca9e876714300a5a0 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:35 +0100 Subject: [PATCH 16/80] Remove former classpath command (now fetch -p), get sources / javadoc via classifiers --- .../main/scala/coursier/cli/Coursier.scala | 40 ++++++++----------- cli/src/main/scala/coursier/cli/Helper.scala | 16 ++++++-- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index d70447079..335f240f1 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -66,19 +66,28 @@ case class Fetch( @HelpMessage("Fetch javadoc artifacts") @ExtraName("D") javadoc: Boolean, + @HelpMessage("Print java -cp compatible output") + @ExtraName("p") + classpath: Boolean, @Recurse common: CommonOptions ) extends CoursierCommand { val helper = new Helper(common, remainingArgs) - val files0 = helper.fetch(main = true, sources = false, javadoc = false) + val files0 = helper.fetch(sources = false, javadoc = false) - println( - files0 - .map(_.toString) - .mkString("\n") - ) + val out = + if (classpath) + files0 + .map(_.toString) + .mkString(File.pathSeparator) + else + files0 + .map(_.toString) + .mkString("\n") + + println(out) } @@ -101,7 +110,7 @@ case class Launch( val helper = new Helper(common, rawDependencies) - val files0 = helper.fetch(main = true, sources = false, javadoc = false) + val files0 = helper.fetch(sources = false, javadoc = false) val cl = new URLClassLoader( files0.map(_.toURI.toURL).toArray, @@ -169,23 +178,6 @@ case class Launch( method.invoke(null, extraArgs.toArray) } -case class Classpath( - @Recurse - common: CommonOptions -) extends CoursierCommand { - - val helper = new Helper(common, remainingArgs) - - val files0 = helper.fetch(main = true, sources = false, javadoc = false) - - Console.out.println( - files0 - .map(_.toString) - .mkString(File.pathSeparator) - ) - -} - case class Bootstrap( @ExtraName("M") @ExtraName("main") diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 20371b125..11a7176b4 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -292,7 +292,7 @@ class Helper( } } - def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = { + def fetch(sources: Boolean, javadoc: Boolean): Seq[File] = { if (verbose0 >= 0) { val msg = cachePolicies match { case Seq(CachePolicy.LocalOnly) => @@ -303,8 +303,18 @@ class Helper( errPrintln(msg) } - val artifacts0 = res.artifacts - val main0 = main || (!sources && !javadoc) + val artifacts0 = + if (sources || javadoc) { + var classifiers = Seq.empty[String] + if (sources) + classifiers = classifiers :+ "sources" + if (javadoc) + classifiers = classifiers :+ "javadoc" + + res.classifiersArtifacts(classifiers) + } else + res.artifacts + val main0 = !sources && !javadoc val artifacts = artifacts0.flatMap{ artifact => var l = List.empty[Artifact] if (sources) From 5c7b1f848d1d9ba6449eebe00deebc3a5ab50335 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:35 +0100 Subject: [PATCH 17/80] Add coursierVerbosity key --- .../main/scala/coursier/CoursierPlugin.scala | 21 +++++++++++++------ plugin/src/main/scala/coursier/Keys.scala | 2 ++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 182de2b7d..4835282af 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -22,6 +22,7 @@ object CoursierPlugin extends AutoPlugin { val coursierMaxIterations = Keys.coursierMaxIterations val coursierChecksums = Keys.coursierChecksums val coursierCachePolicy = Keys.coursierCachePolicy + val coursierVerbosity = Keys.coursierVerbosity val coursierResolvers = Keys.coursierResolvers val coursierCache = Keys.coursierCache val coursierProject = Keys.coursierProject @@ -59,6 +60,8 @@ object CoursierPlugin extends AutoPlugin { val resolvers = coursierResolvers.value + val verbosity = coursierVerbosity.value + val startRes = Resolution( currentProject.dependencies.map { case (_, dep) => dep }.toSet, @@ -87,9 +90,11 @@ object CoursierPlugin extends AutoPlugin { s"${dep.module}:${dep.version}:$config->${dep.configuration}" }.sorted - errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") - for (depRepr <- depsRepr) - errPrintln(s" $depRepr") + if (verbosity >= 0) + errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") + if (verbosity >= 1) + for (depRepr <- depsRepr) + errPrintln(s" $depRepr") val res = startRes .process @@ -101,7 +106,8 @@ object CoursierPlugin extends AutoPlugin { if (!res.isDone) throw new Exception(s"Maximum number of iteration reached!") - errPrintln("Resolution done") + if (verbosity >= 0) + errPrintln("Resolution done") def repr(dep: Dependency) = { // dep.version can be an interval, whereas the one from project can't @@ -148,14 +154,16 @@ object CoursierPlugin extends AutoPlugin { files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) } - errPrintln(s"Fetching artifacts") + if (verbosity >= 0) + errPrintln(s"Fetching artifacts") // rename val trDepsWithArtifacts = Task.gatherUnordered(trDepsWithArtifactsTasks).attemptRun match { case -\/(ex) => throw new Exception(s"Error while downloading / verifying artifacts", ex) case \/-(l) => l.toMap } - errPrintln(s"Fetching artifacts: done") + if (verbosity >= 0) + errPrintln(s"Fetching artifacts: done") val configs = ivyConfigurations.value.map(c => c.name -> c.extendsConfigs.map(_.name)).toMap def allExtends(c: String) = { @@ -239,6 +247,7 @@ object CoursierPlugin extends AutoPlugin { coursierMaxIterations := 50, coursierChecksums := Seq(Some("SHA-1"), Some("MD5")), coursierCachePolicy := CachePolicy.FetchMissing, + coursierVerbosity := 0, coursierResolvers <<= Tasks.coursierResolversTask, coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), update <<= task, diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala index cf6c56990..ba66e8178 100644 --- a/plugin/src/main/scala/coursier/Keys.scala +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -9,6 +9,8 @@ object Keys { val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "") //Seq(Some("SHA-1"), Some("MD5")) val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "") // = CachePolicy.FetchMissing + val coursierVerbosity = SettingKey[Int]("coursier-verbosity", "") + val coursierResolvers = TaskKey[Seq[Resolver]]("coursier-resolvers", "") val coursierCache = SettingKey[File]("coursier-cache", "") From d5d28a0615b2bc93dc6141d778235a8c94a1cddf Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:36 +0100 Subject: [PATCH 18/80] Fixup (rework CLI) --- cli/src/main/scala/coursier/cli/Coursier.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 335f240f1..dce73b2f4 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -15,7 +15,7 @@ case class CommonOptions( @HelpMessage("Download mode (default: missing, that is fetch things missing from cache)") @ValueDescription("offline|update-changing|update|missing|force") @ExtraName("m") - mode: String, + mode: String = "missing", @HelpMessage("Quiet output") @ExtraName("q") quiet: Boolean, From c3cc002be2d78905407539eba9bf071c310fcd16 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:36 +0100 Subject: [PATCH 19/80] fixup (Better handling of classifiers) --- .../src/main/scala/coursier/maven/MavenRepository.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index d05fcf472..f549bb73b 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -246,7 +246,11 @@ case class MavenRepository( proj <- Pom.project(xml) } yield proj.copy( configurations = defaultConfigurations, - publications = ??? + publications = Seq( + "compile" -> Publication(module.name, "jar", "jar", ""), + "docs" -> Publication(module.name, "doc", "jar", "javadoc"), + "sources" -> Publication(module.name, "src", "jar", "sources") + ) )): (String \/ Project) } } From 22b41317d249216ef9cb902f9022d5dbc009f331 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:36 +0100 Subject: [PATCH 20/80] Ensure sources and javadoc fetching are fine --- cli/src/main/scala/coursier/cli/Coursier.scala | 2 +- cli/src/main/scala/coursier/cli/Helper.scala | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index dce73b2f4..b576c79ff 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -75,7 +75,7 @@ case class Fetch( val helper = new Helper(common, remainingArgs) - val files0 = helper.fetch(sources = false, javadoc = false) + val files0 = helper.fetch(sources = sources, javadoc = javadoc) val out = if (classpath) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 11a7176b4..420d7efdc 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -303,7 +303,7 @@ class Helper( errPrintln(msg) } - val artifacts0 = + val artifacts = if (sources || javadoc) { var classifiers = Seq.empty[String] if (sources) @@ -314,18 +314,6 @@ class Helper( res.classifiersArtifacts(classifiers) } else res.artifacts - val main0 = !sources && !javadoc - val artifacts = artifacts0.flatMap{ artifact => - var l = List.empty[Artifact] - if (sources) - l = artifact.extra.get("sources").toList ::: l - if (javadoc) - l = artifact.extra.get("javadoc").toList ::: l - if (main0) - l = artifact :: l - - l - } val logger = if (verbose0 >= 0) From cdf60562f739af3bfa1252630222289464f80924 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:36 +0100 Subject: [PATCH 21/80] Fix --- plugin/src/main/scala/coursier/ToSbt.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/src/main/scala/coursier/ToSbt.scala b/plugin/src/main/scala/coursier/ToSbt.scala index 6a83eaa78..4f003d4f4 100644 --- a/plugin/src/main/scala/coursier/ToSbt.scala +++ b/plugin/src/main/scala/coursier/ToSbt.scala @@ -6,10 +6,10 @@ object ToSbt { def artifact(module: Module, artifact: Artifact): sbt.Artifact = sbt.Artifact( - s"${module.organization}:${module.name}", + module.name, artifact.attributes.`type`, "jar", - Some(artifact.attributes.classifier), + Some(artifact.attributes.classifier).filter(_.nonEmpty), Nil, Some(url(artifact.url)), Map.empty From a3188a9b8aff6c6fdc2b5aca263f650d9d16beca Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:37 +0100 Subject: [PATCH 22/80] Override updateClassifiers task --- .../main/scala/coursier/core/Resolution.scala | 10 +++++-- .../main/scala/coursier/CoursierPlugin.scala | 30 +++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index 4b9d16e15..ec38adc46 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -762,16 +762,22 @@ case class Resolution( def artifacts: Seq[Artifact] = artifacts0(None) - def dependencyArtifacts: Seq[(Dependency, Artifact)] = + private def dependencyArtifacts0(overrideClassifiers: Option[Seq[String]]): Seq[(Dependency, Artifact)] = for { dep <- minDependencies.toSeq (source, proj) <- projectCache .get(dep.moduleVersion) .toSeq artifact <- source - .artifacts(dep, proj, None) + .artifacts(dep, proj, overrideClassifiers) } yield dep -> artifact + def dependencyArtifacts: Seq[(Dependency, Artifact)] = + dependencyArtifacts0(None) + + def dependencyClassifiersArtifacts(classifiers: Seq[String]): Seq[(Dependency, Artifact)] = + dependencyArtifacts0(Some(classifiers)) + def errors: Seq[(Dependency, Seq[String])] = for { dep <- dependencies.toSeq diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 4835282af..69cceecc3 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -44,7 +44,8 @@ object CoursierPlugin extends AutoPlugin { } - private def task = Def.task { + private def updateTask(withClassifiers: Boolean) = Def.task { + // let's update only one module at once, for a better output // Downloads are already parallel, no need to parallelize further anyway synchronized { @@ -148,7 +149,19 @@ object CoursierPlugin extends AutoPlugin { } } - val trDepsWithArtifactsTasks = res.artifacts + val classifiers = + if (withClassifiers) + Some(transitiveClassifiers.value) + else + None + + val allArtifacts = + classifiers match { + case None => res.artifacts + case Some(cl) => res.classifiersArtifacts(cl) + } + + val trDepsWithArtifactsTasks = allArtifacts .toVector .map { a => files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) @@ -188,8 +201,14 @@ object CoursierPlugin extends AutoPlugin { val sbtModuleReportsPerScope = configs.map { case (c, _) => c -> { val a = allExtends(c).flatMap(depsByConfig.getOrElse(_, Nil)) - res.part(a) - .dependencyArtifacts + val partialRes = res.part(a) + val depArtifacts = + classifiers match { + case None => partialRes.dependencyArtifacts + case Some(cl) => partialRes.dependencyClassifiersArtifacts(cl) + } + + depArtifacts .groupBy { case (dep, _) => dep } .map { case (dep, l) => dep -> l.map { case (_, a) => a } } .map { case (dep, artifacts) => @@ -250,7 +269,8 @@ object CoursierPlugin extends AutoPlugin { coursierVerbosity := 0, coursierResolvers <<= Tasks.coursierResolversTask, coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), - update <<= task, + update <<= updateTask(withClassifiers = false), + updateClassifiers <<= updateTask(withClassifiers = true), coursierProject <<= Tasks.coursierProjectTask, coursierProjects <<= Tasks.coursierProjectsTask ) From f52e2ecca43a0dfb27cf8a966d584afbba90b067 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:37 +0100 Subject: [PATCH 23/80] Add support for attributes --- cli/src/main/scala/coursier/cli/Helper.scala | 18 +++++++++++++++-- .../coursier/core/compatibility/package.scala | 2 +- .../coursier/core/compatibility/package.scala | 12 ++++++++--- .../scala/coursier/core/Definitions.scala | 14 +++++++++---- .../scala/coursier/ivy/IvyRepository.scala | 20 +++++++++---------- .../src/main/scala/coursier/ivy/IvyXml.scala | 8 ++++++-- .../src/main/scala/coursier/maven/Pom.scala | 4 ++-- .../src/main/scala/coursier/package.scala | 4 ++-- .../src/main/scala/coursier/util/Xml.scala | 11 ++++++++-- web/src/main/scala/coursier/web/Backend.scala | 5 ++++- 10 files changed, 69 insertions(+), 29 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 420d7efdc..ce4db9ea2 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -158,8 +158,22 @@ class Helper( } val moduleVersions = splitDependencies.map{ - case Seq(org, name, version) => - (Module(org, name), version) + case Seq(org, namePart, version) => + val p = namePart.split(';') + val name = p.head + val splitAttributes = p.tail.map(_.split("=", 2).toSeq).toSeq + val malformedAttributes = splitAttributes.filter(_.length != 2) + if (malformedAttributes.nonEmpty) { + // FIXME Get these for all dependencies at once + Console.err.println(s"Malformed attributes in ${splitDependencies.mkString(":")}") + // :( + sys.exit(255) + } + val attributes = splitAttributes.collect { + case Seq(k, v) => k -> v + } + println(s"-> $org:$name:$attributes") + (Module(org, name, attributes.toMap), version) } val deps = moduleVersions.map{case (mod, ver) => diff --git a/core/js/src/main/scala/coursier/core/compatibility/package.scala b/core/js/src/main/scala/coursier/core/compatibility/package.scala index f7659c987..b168d85e1 100644 --- a/core/js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/js/src/main/scala/coursier/core/compatibility/package.scala @@ -51,7 +51,7 @@ package object compatibility { .map(l => List.tabulate(l.length)(l.item).map(fromNode)) .getOrElse(Nil) - def attributes: Seq[(String, String)] = ??? + def attributes = ??? // `exists` instead of `contains`, for scala 2.10 def isText = diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index 1b398bcab..3a8f6dc49 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -2,7 +2,7 @@ package coursier.core import coursier.util.Xml -import scala.xml.{ MetaData, Null } +import scala.xml.{ Attribute, MetaData, Null } package object compatibility { @@ -19,14 +19,20 @@ package object compatibility { def fromNode(node: scala.xml.Node): Xml.Node = new Xml.Node { lazy val attributes = { - def helper(m: MetaData): Stream[(String, String)] = + def helper(m: MetaData): Stream[(String, String, String)] = m match { case Null => Stream.empty case attr => + val pre = attr match { + case a: Attribute => Option(node.getNamespace(a.pre)).getOrElse("") + case _ => "" + } + val value = attr.value.collect { case scala.xml.Text(t) => t }.mkString("") - (attr.key -> value) #:: helper(m.next) + + (pre, attr.key, value) #:: helper(m.next) } helper(node.attributes).toVector diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index c741d6a2d..08a247bf7 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -8,12 +8,11 @@ package coursier.core * between them. * * Using the same terminology as Ivy. - * - * Ivy attributes would land here, if support for it is added. */ case class Module( organization: String, - name: String + name: String, + attributes: Map[String, String] ) { def trim: Module = copy( @@ -21,7 +20,14 @@ case class Module( name = name.trim ) - override def toString = s"$organization:$name" + private def attributesStr = attributes.toSeq + .sortBy { case (k, _) => k } + .map { case (k, v) => s"$k=$v" } + .mkString(";") + + override def toString = + s"$organization:$name" + + (if (attributes.nonEmpty) s";$attributesStr" else "") } /** diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index b1ce1a788..9a4e0c3fc 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -145,23 +145,22 @@ case class IvyRepository( // list of variables that should be supported. // Some are missing (branch, conf, originalName). private def variables( - org: String, - name: String, + module: Module, version: String, `type`: String, artifact: String, ext: String ) = Map( - "organization" -> org, - "organisation" -> org, - "orgPath" -> org.replace('.', '/'), - "module" -> name, + "organization" -> module.organization, + "organisation" -> module.organization, + "orgPath" -> module.organization.replace('.', '/'), + "module" -> module.name, "revision" -> version, "type" -> `type`, "artifact" -> artifact, "ext" -> ext - ) + ) ++ module.attributes val source: Artifact.Source = new Artifact.Source { @@ -191,8 +190,7 @@ case class IvyRepository( val retainedWithUrl = retained.flatMap { p => substitute(variables( - dependency.module.organization, - dependency.module.name, + dependency.module, dependency.version, p.`type`, p.name, @@ -225,7 +223,9 @@ case class IvyRepository( val eitherArtifact: String \/ Artifact = for { - url <- substitute(variables(module.organization, module.name, version, "ivy", "ivy", "xml")) + url <- substitute( + variables(module, version, "ivy", "ivy", "xml") + ) } yield Artifact( url, diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala index 62b70cb8d..7725817bc 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -7,12 +7,15 @@ import scalaz.{ Node => _, _ }, Scalaz._ object IvyXml { + val attributesNamespace = "http://ant.apache.org/ivy/extra" + private def info(node: Node): String \/ (Module, String) = for { org <- node.attribute("organisation") name <- node.attribute("module") version <- node.attribute("revision") - } yield (Module(org, name), version) + attr = node.attributesFromNamespace(attributesNamespace) + } yield (Module(org, name, attr.toMap), version) // FIXME Errors are ignored here private def configurations(node: Node): Seq[(String, Seq[String])] = @@ -53,8 +56,9 @@ object IvyXml { (fromConf, toConf) <- rawConf.split(',').toSeq.map(_.split("->", 2)).collect { case Array(from, to) => from -> to } + attr = node.attributesFromNamespace(attributesNamespace) } yield fromConf -> Dependency( - Module(org, name), + Module(org, name, attr.toMap), version, toConf, allConfsExcludes ++ excludes.getOrElse(fromConf, Set.empty), diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 6ad669204..f6980bc11 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -22,7 +22,7 @@ object Pom { else e } name <- text(node, "artifactId", "Name") - } yield Module(organization, name).trim + } yield Module(organization, name, Map.empty).trim } private def readVersion(node: Node) = @@ -307,7 +307,7 @@ object Pom { .toList .traverseU(snapshotVersion) } yield SnapshotVersioning( - Module(organization, name), + Module(organization, name, Map.empty), version, latest, release, diff --git a/core/shared/src/main/scala/coursier/package.scala b/core/shared/src/main/scala/coursier/package.scala index 3a7125b49..ec15aabc6 100644 --- a/core/shared/src/main/scala/coursier/package.scala +++ b/core/shared/src/main/scala/coursier/package.scala @@ -42,8 +42,8 @@ package object coursier { type Module = core.Module object Module { - def apply(organization: String, name: String): Module = - core.Module(organization, name) + def apply(organization: String, name: String, attributes: Map[String, String] = Map.empty): Module = + core.Module(organization, name, attributes) } type ModuleVersion = (core.Module, String) diff --git a/core/shared/src/main/scala/coursier/util/Xml.scala b/core/shared/src/main/scala/coursier/util/Xml.scala index a4e9c86df..83fe20f9b 100644 --- a/core/shared/src/main/scala/coursier/util/Xml.scala +++ b/core/shared/src/main/scala/coursier/util/Xml.scala @@ -7,13 +7,20 @@ object Xml { /** A representation of an XML node/document, with different implementations on JVM and JS */ trait Node { def label: String - def attributes: Seq[(String, String)] + /** Namespace / key / value */ + def attributes: Seq[(String, String, String)] def children: Seq[Node] def isText: Boolean def textContent: String def isElement: Boolean - lazy val attributesMap = attributes.toMap + def attributesFromNamespace(namespace: String): Seq[(String, String)] = + attributes.collect { + case (`namespace`, k, v) => + k -> v + } + + lazy val attributesMap = attributes.map { case (_, k, v) => k -> v }.toMap def attribute(name: String): String \/ String = attributesMap.get(name) match { case None => -\/(s"Missing attribute $name") diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 5a03d9b97..69e6c3d96 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -406,7 +406,10 @@ object App { } val sortedDeps = res.minDependencies.toList - .sortBy(dep => coursier.core.Module.unapply(dep.module).get) + .sortBy { dep => + val (org, name, _) = coursier.core.Module.unapply(dep.module).get + (org, name) + } <.table(^.`class` := "table", <.thead( From 8af6efa708603a50b0896ffbd2f5fd61123bd758 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:37 +0100 Subject: [PATCH 24/80] Verbosity fix --- plugin/src/main/scala/coursier/CoursierPlugin.scala | 6 ++++++ plugin/src/main/scala/coursier/InterProjectRepository.scala | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 69cceecc3..0dc05fa49 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -70,6 +70,12 @@ object CoursierPlugin extends AutoPlugin { forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap ) + if (verbosity >= 1) { + println("InterProjectRepository") + for ((p, _) <- projects) + println(s" ${p.module}:${p.version}") + } + val interProjectRepo = InterProjectRepository(projects) val repositories = interProjectRepo +: resolvers.flatMap(FromSbt.repository(_, ivyProperties)) diff --git a/plugin/src/main/scala/coursier/InterProjectRepository.scala b/plugin/src/main/scala/coursier/InterProjectRepository.scala index 62b086989..aa833f4f6 100644 --- a/plugin/src/main/scala/coursier/InterProjectRepository.scala +++ b/plugin/src/main/scala/coursier/InterProjectRepository.scala @@ -22,10 +22,6 @@ case class InterProjectSource(artifacts: Map[(Module, String), Map[String, Seq[A case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artifact])])]) extends Repository { - Console.err.println("InterProjectRepository") - for ((p, _) <- projects) - Console.err.println(s" ${p.module}:${p.version}") - private val map = projects .map { case (proj, a) => proj.moduleVersion -> proj } .toMap From d188cb0b1a2ce274073dd381eabcdf8e2311e8f4 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:37 +0100 Subject: [PATCH 25/80] Fixup (attributes support) --- plugin/src/main/scala/coursier/FromSbt.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index b7ea73e44..761979d07 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -39,7 +39,7 @@ object FromSbt { val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) val dep = Dependency( - Module(module.organization, fullName), + Module(module.organization, fullName, module.extraAttributes), module.revision, exclusions = module.exclusions.map { rule => // FIXME Other `rule` fields are ignored here @@ -79,7 +79,11 @@ object FromSbt { val deps = allDependencies.flatMap(dependencies(_, scalaVersion, scalaBinaryVersion)) Project( - Module(projectID.organization, sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion)), + Module( + projectID.organization, + sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion), + projectID.extraAttributes + ), projectID.revision, deps, ivyConfigurations, From 5d32358c2eaac54dde45ddf98040be598f021a40 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:38 +0100 Subject: [PATCH 26/80] Better handling of attributes from SBT --- .../main/scala/coursier/cli/Coursier.scala | 2 ++ cli/src/main/scala/coursier/cli/Helper.scala | 11 ++++++++- .../coursier/maven/MavenRepository.scala | 23 +++++++++++++++---- plugin/src/main/scala/coursier/FromSbt.scala | 11 ++++++--- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index b576c79ff..1fcbbcf8f 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -30,6 +30,8 @@ case class CommonOptions( repository: List[String], @HelpMessage("Do not add default repositories (~/.ivy2/local, and Central)") noDefault: Boolean = false, + @HelpMessage("Modify names in Maven repository paths for SBT plugins") + sbtPluginHack: Boolean = false, @HelpMessage("Force module version") @ValueDescription("organization:name:forcedVersion") @ExtraName("V") diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index ce4db9ea2..8ec1f43ef 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -114,10 +114,19 @@ class Helper( sys.exit(255) } - val repositories = + val repositories1 = (if (common.noDefault) Nil else defaultRepositories) ++ repositories0.collect { case Right(r) => r } + val repositories = + if (common.sbtPluginHack) + repositories1.map { + case m: MavenRepository => m.copy(sbtAttrStub = true) + case other => other + } + else + repositories1 + val (rawDependencies, extraArgs) = { val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) idxOpt.fold((remainingArgs, Seq.empty[String])) { idx => diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index f549bb73b..1ab35e5e8 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -49,7 +49,9 @@ object MavenRepository { case class MavenRepository( root: String, ivyLike: Boolean = false, - changing: Option[Boolean] = None + changing: Option[Boolean] = None, + /** Hackish hack for sbt plugins mainly - what this does really sucks */ + sbtAttrStub: Boolean = false ) extends Repository { import Repository._ @@ -58,6 +60,17 @@ case class MavenRepository( val root0 = if (root.endsWith("/")) root else root + "/" val source = MavenSource(root0, ivyLike) + private def dirModuleName(module: Module): String = + if (sbtAttrStub) { + var name = module.name + for (scalaVersion <- module.attributes.get("scalaVersion")) + name = name + "_" + scalaVersion + for (sbtVersion <- module.attributes.get("sbtVersion")) + name = name + "_" + sbtVersion + name + } else + module.name + def projectArtifact( module: Module, version: String, @@ -68,7 +81,7 @@ case class MavenRepository( if (ivyLike) ivyLikePath( module.organization, - module.name, + dirModuleName(module), // maybe not what we should do here, don't know versioningValue getOrElse version, "poms", "", @@ -76,7 +89,7 @@ case class MavenRepository( ) else module.organization.split('.').toSeq ++ Seq( - module.name, + dirModuleName(module), version, s"${module.name}-${versioningValue getOrElse version}.pom" ) @@ -98,7 +111,7 @@ case class MavenRepository( else { val path = ( module.organization.split('.').toSeq ++ Seq( - module.name, + dirModuleName(module), "maven-metadata.xml" ) ) .map(encodeURIComponent) @@ -125,7 +138,7 @@ case class MavenRepository( else { val path = ( module.organization.split('.').toSeq ++ Seq( - module.name, + dirModuleName(module), version, "maven-metadata.xml" ) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 761979d07..2e8dec8e1 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -28,6 +28,11 @@ object FromSbt { } yield (from, to) } + def attributes(attr: Map[String, String]): Map[String, String] = + attr.map { case (k, v) => + k.stripPrefix("e:") -> v + } + def dependencies( module: ModuleID, scalaVersion: String, @@ -39,7 +44,7 @@ object FromSbt { val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) val dep = Dependency( - Module(module.organization, fullName, module.extraAttributes), + Module(module.organization, fullName, FromSbt.attributes(module.extraAttributes)), module.revision, exclusions = module.exclusions.map { rule => // FIXME Other `rule` fields are ignored here @@ -82,7 +87,7 @@ object FromSbt { Module( projectID.organization, sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion), - projectID.extraAttributes + FromSbt.attributes(projectID.extraAttributes) ), projectID.revision, deps, @@ -102,7 +107,7 @@ object FromSbt { case sbt.MavenRepository(_, root) => if (root.startsWith("http://") || root.startsWith("https://")) { val root0 = if (root.endsWith("/")) root else root + "/" - Some(MavenRepository(root0)) + Some(MavenRepository(root0, sbtAttrStub = true)) } else { Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it") None From c42682ed60556c704271aec930555e5ff749446e Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:38 +0100 Subject: [PATCH 27/80] Fixup (better handling of sbt attributes) --- .../src/main/scala/coursier/Fetch.scala | 30 +++++++++++++--- .../coursier/maven/MavenRepository.scala | 36 ++++++++++--------- .../scala/coursier/maven/MavenSource.scala | 7 ++-- 3 files changed, 50 insertions(+), 23 deletions(-) diff --git a/core/shared/src/main/scala/coursier/Fetch.scala b/core/shared/src/main/scala/coursier/Fetch.scala index 7bd8a15e9..65d393ad3 100644 --- a/core/shared/src/main/scala/coursier/Fetch.scala +++ b/core/shared/src/main/scala/coursier/Fetch.scala @@ -1,5 +1,7 @@ package coursier +import coursier.maven.MavenSource + import scalaz._ object Fetch { @@ -39,11 +41,21 @@ object Fetch { val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { case (acc, (repo, eitherProjTask)) => + val looseModuleValidation = repo match { + case m: MavenRepository => m.sbtAttrStub // that sucks so much + case _ => false + } + val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module F.bind(acc) { case -\/(errors) => F.map(eitherProjTask)(_.flatMap{case (source, project) => - if (project.module == module) \/-((source, project)) - else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") + val projModule = + if (looseModuleValidation) + project.module.copy(attributes = Map.empty) + else + project.module + if (projModule == moduleCmp) \/-((source, project)) + else -\/(s"Wrong module returned (expected: $moduleCmp, got: ${project.module})") }.leftMap(error => error +: errors)) case res @ \/-(_) => @@ -52,8 +64,18 @@ object Fetch { } EitherT(F.map(task)(_.leftMap(_.reverse))) - .map {case x @ (_, proj) => - assert(proj.module == module) + .map {case x @ (source, proj) => + val looseModuleValidation = source match { + case m: MavenSource => m.sbtAttrStub // omfg + case _ => false + } + val projModule = + if (looseModuleValidation) + proj.module.copy(attributes = Map.empty) + else + proj.module + val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module + assert(projModule == moduleCmp) x } } diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 1ab35e5e8..821cd1866 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -11,6 +11,7 @@ object MavenRepository { def ivyLikePath( org: String, + dirName: String, name: String, version: String, subDir: String, @@ -19,7 +20,7 @@ object MavenRepository { ) = Seq( org, - name, + dirName, version, subDir, s"$name$baseSuffix.$ext" @@ -44,6 +45,17 @@ object MavenRepository { "test" -> Seq("runtime") ) + def dirModuleName(module: Module, sbtAttrStub: Boolean): String = + if (sbtAttrStub) { + var name = module.name + for (scalaVersion <- module.attributes.get("scalaVersion")) + name = name + "_" + scalaVersion + for (sbtVersion <- module.attributes.get("sbtVersion")) + name = name + "_" + sbtVersion + name + } else + module.name + } case class MavenRepository( @@ -58,18 +70,7 @@ case class MavenRepository( import MavenRepository._ val root0 = if (root.endsWith("/")) root else root + "/" - val source = MavenSource(root0, ivyLike) - - private def dirModuleName(module: Module): String = - if (sbtAttrStub) { - var name = module.name - for (scalaVersion <- module.attributes.get("scalaVersion")) - name = name + "_" + scalaVersion - for (sbtVersion <- module.attributes.get("sbtVersion")) - name = name + "_" + sbtVersion - name - } else - module.name + val source = MavenSource(root0, ivyLike, changing, sbtAttrStub) def projectArtifact( module: Module, @@ -81,7 +82,8 @@ case class MavenRepository( if (ivyLike) ivyLikePath( module.organization, - dirModuleName(module), // maybe not what we should do here, don't know + dirModuleName(module, sbtAttrStub), // maybe not what we should do here, don't know + module.name, versioningValue getOrElse version, "poms", "", @@ -89,7 +91,7 @@ case class MavenRepository( ) else module.organization.split('.').toSeq ++ Seq( - dirModuleName(module), + dirModuleName(module, sbtAttrStub), version, s"${module.name}-${versioningValue getOrElse version}.pom" ) @@ -111,7 +113,7 @@ case class MavenRepository( else { val path = ( module.organization.split('.').toSeq ++ Seq( - dirModuleName(module), + dirModuleName(module, sbtAttrStub), "maven-metadata.xml" ) ) .map(encodeURIComponent) @@ -138,7 +140,7 @@ case class MavenRepository( else { val path = ( module.organization.split('.').toSeq ++ Seq( - dirModuleName(module), + dirModuleName(module, sbtAttrStub), version, "maven-metadata.xml" ) diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 467c8584a..f07a4f5b3 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -5,7 +5,9 @@ import coursier.core._ case class MavenSource( root: String, ivyLike: Boolean, - changing: Option[Boolean] = None + changing: Option[Boolean] = None, + /** See doc on MavenRepository */ + sbtAttrStub: Boolean ) extends Artifact.Source { import Repository._ @@ -47,6 +49,7 @@ case class MavenSource( def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = ivyLikePath( module.organization, + MavenRepository.dirModuleName(module, sbtAttrStub), module.name, project.version, subDir, @@ -66,7 +69,7 @@ case class MavenSource( ) module.organization.split('.').toSeq ++ Seq( - module.name, + MavenRepository.dirModuleName(module, sbtAttrStub), project.version, s"${module.name}-${versioning getOrElse project.version}${Some(publication.classifier).filter(_.nonEmpty).map("-" + _).mkString}.${publication.ext}" ) From ef31d8344aa65cfcc0a3b842198017f0f57a510f Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:38 +0100 Subject: [PATCH 28/80] global-plugins - attempt #1 --- .../main/scala/coursier/CoursierPlugin.scala | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 0dc05fa49..9ee64f198 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -17,6 +17,33 @@ object CoursierPlugin extends AutoPlugin { private def errPrintln(s: String): Unit = scala.Console.err.println(s) + // org.scala-sbt:global-plugins;sbtVersion=0.13;scalaVersion=2.10:0.0 + private val globalPluginsProject = Project( + Module("org.scala-sbt", "global-plugins", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10")), + "0.0", + Nil, + Map.empty, + None, + Nil, + Map.empty, + Nil, + None, + None, + Nil + ) + + private val globalPluginsArtifacts = Seq( + "" -> Seq( + Artifact( + new File(sys.props("user.home") + "/.sbt/0.13/plugins/target") .toURI.toString, + Map.empty, + Map.empty, + Attributes(), + changing = true + ) + ) + ) + object autoImport { val coursierParallelDownloads = Keys.coursierParallelDownloads val coursierMaxIterations = Keys.coursierMaxIterations @@ -64,19 +91,21 @@ object CoursierPlugin extends AutoPlugin { val verbosity = coursierVerbosity.value + val projects0 = projects :+ (globalPluginsProject -> globalPluginsArtifacts) + val startRes = Resolution( currentProject.dependencies.map { case (_, dep) => dep }.toSet, filter = Some(dep => !dep.optional), - forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap + forceVersions = projects0.map { case (proj, _) => proj.moduleVersion }.toMap ) if (verbosity >= 1) { println("InterProjectRepository") - for ((p, _) <- projects) + for ((p, _) <- projects0) println(s" ${p.module}:${p.version}") } - val interProjectRepo = InterProjectRepository(projects) + val interProjectRepo = InterProjectRepository(projects0) val repositories = interProjectRepo +: resolvers.flatMap(FromSbt.repository(_, ivyProperties)) val files = Files( @@ -272,7 +301,7 @@ object CoursierPlugin extends AutoPlugin { coursierMaxIterations := 50, coursierChecksums := Seq(Some("SHA-1"), Some("MD5")), coursierCachePolicy := CachePolicy.FetchMissing, - coursierVerbosity := 0, + coursierVerbosity := 1, coursierResolvers <<= Tasks.coursierResolversTask, coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), update <<= updateTask(withClassifiers = false), From 4aed6f6c38215afa93ba664031a9ff2479165bd3 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:38 +0100 Subject: [PATCH 29/80] global-plugins - attempt #2 - success --- .../scala/coursier/ivy/IvyRepository.scala | 29 ++++++++---- .../main/scala/coursier/CoursierPlugin.scala | 47 +++++-------------- 2 files changed, 34 insertions(+), 42 deletions(-) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 9a4e0c3fc..2fe8a20b9 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -80,7 +80,9 @@ object IvyRepository { case class IvyRepository( pattern: String, changing: Option[Boolean] = None, - properties: Map[String, String] = Map.empty + properties: Map[String, String] = Map.empty, + withChecksums: Boolean = true, + withSignatures: Boolean = true ) extends Repository { import Repository._ @@ -199,15 +201,20 @@ case class IvyRepository( } retainedWithUrl.map { case (p, url) => - Artifact( + var artifact = Artifact( url, Map.empty, Map.empty, p.attributes, changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable ) - .withDefaultChecksums - .withDefaultSignature + + if (withChecksums) + artifact = artifact.withDefaultChecksums + if (withSignatures) + artifact = artifact.withDefaultSignature + + artifact } } } @@ -226,16 +233,22 @@ case class IvyRepository( url <- substitute( variables(module, version, "ivy", "ivy", "xml") ) - } yield - Artifact( + } yield { + var artifact = Artifact( url, Map.empty, Map.empty, Attributes("ivy", ""), changing = changing.getOrElse(version.contains("-SNAPSHOT")) ) - .withDefaultChecksums - .withDefaultSignature + + if (withChecksums) + artifact = artifact.withDefaultChecksums + if (withSignatures) + artifact = artifact.withDefaultSignature + + artifact + } for { artifact <- EitherT(F.point(eitherArtifact)) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 9ee64f198..c6b2109c6 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -3,6 +3,7 @@ package coursier import java.io.{ File, OutputStreamWriter } import coursier.cli.TermDisplay +import coursier.ivy.IvyRepository import sbt.{ MavenRepository => _, _ } import sbt.Keys._ @@ -17,33 +18,6 @@ object CoursierPlugin extends AutoPlugin { private def errPrintln(s: String): Unit = scala.Console.err.println(s) - // org.scala-sbt:global-plugins;sbtVersion=0.13;scalaVersion=2.10:0.0 - private val globalPluginsProject = Project( - Module("org.scala-sbt", "global-plugins", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10")), - "0.0", - Nil, - Map.empty, - None, - Nil, - Map.empty, - Nil, - None, - None, - Nil - ) - - private val globalPluginsArtifacts = Seq( - "" -> Seq( - Artifact( - new File(sys.props("user.home") + "/.sbt/0.13/plugins/target") .toURI.toString, - Map.empty, - Map.empty, - Attributes(), - changing = true - ) - ) - ) - object autoImport { val coursierParallelDownloads = Keys.coursierParallelDownloads val coursierMaxIterations = Keys.coursierMaxIterations @@ -91,22 +65,27 @@ object CoursierPlugin extends AutoPlugin { val verbosity = coursierVerbosity.value - val projects0 = projects :+ (globalPluginsProject -> globalPluginsArtifacts) - val startRes = Resolution( currentProject.dependencies.map { case (_, dep) => dep }.toSet, filter = Some(dep => !dep.optional), - forceVersions = projects0.map { case (proj, _) => proj.moduleVersion }.toMap + forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap ) if (verbosity >= 1) { println("InterProjectRepository") - for ((p, _) <- projects0) + for ((p, _) <- projects) println(s" ${p.module}:${p.version}") } - val interProjectRepo = InterProjectRepository(projects0) - val repositories = interProjectRepo +: resolvers.flatMap(FromSbt.repository(_, ivyProperties)) + val globalPluginsRepo = IvyRepository( + new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString + + "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]", + withChecksums = false, + withSignatures = false + ) + + val interProjectRepo = InterProjectRepository(projects) + val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) val files = Files( Seq("http://" -> new File(cacheDir, "http"), "https://" -> new File(cacheDir, "https")), @@ -299,7 +278,7 @@ object CoursierPlugin extends AutoPlugin { override lazy val projectSettings = Seq( coursierParallelDownloads := 6, coursierMaxIterations := 50, - coursierChecksums := Seq(Some("SHA-1"), Some("MD5")), + coursierChecksums := Seq(Some("SHA-1"), Some("MD5"), None), coursierCachePolicy := CachePolicy.FetchMissing, coursierVerbosity := 1, coursierResolvers <<= Tasks.coursierResolversTask, From a5930d0db828f5f8c3d5dc150ce4e4d0df100272 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:39 +0100 Subject: [PATCH 30/80] Fix --- plugin/src/main/scala/coursier/FromSbt.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 2e8dec8e1..06419193a 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -44,7 +44,7 @@ object FromSbt { val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) val dep = Dependency( - Module(module.organization, fullName, FromSbt.attributes(module.extraAttributes)), + Module(module.organization, fullName, FromSbt.attributes(module.extraDependencyAttributes)), module.revision, exclusions = module.exclusions.map { rule => // FIXME Other `rule` fields are ignored here @@ -87,7 +87,7 @@ object FromSbt { Module( projectID.organization, sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion), - FromSbt.attributes(projectID.extraAttributes) + FromSbt.attributes(projectID.extraDependencyAttributes) ), projectID.revision, deps, From 056f65b90f394dd19d39a29d94c075e1361237e9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:39 +0100 Subject: [PATCH 31/80] No artifacts for global-plugin repo --- .../scala/coursier/core/Definitions.scala | 10 ++ .../scala/coursier/ivy/IvyRepository.scala | 105 +++++++++--------- .../main/scala/coursier/CoursierPlugin.scala | 3 +- 3 files changed, 67 insertions(+), 51 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 08a247bf7..089e0f490 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -164,4 +164,14 @@ object Artifact { overrideClassifiers: Option[Seq[String]] ): Seq[Artifact] } + + object Source { + val empty: Source = new Source { + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ): Seq[Artifact] = Nil + } + } } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 2fe8a20b9..82a77a0d7 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -82,7 +82,8 @@ case class IvyRepository( changing: Option[Boolean] = None, properties: Map[String, String] = Map.empty, withChecksums: Boolean = true, - withSignatures: Boolean = true + withSignatures: Boolean = true, + withArtifacts: Boolean = true ) extends Repository { import Repository._ @@ -165,59 +166,63 @@ case class IvyRepository( ) ++ module.attributes - val source: Artifact.Source = new Artifact.Source { - def artifacts( - dependency: Dependency, - project: Project, - overrideClassifiers: Option[Seq[String]] - ) = { + val source: Artifact.Source = + if (withArtifacts) + new Artifact.Source { + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ) = { - val retained = - overrideClassifiers match { - case None => - project.publications.collect { - case (conf, p) - if conf == "*" || - conf == dependency.configuration || - project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) => - p - } - case Some(classifiers) => - val classifiersSet = classifiers.toSet - project.publications.collect { - case (_, p) if classifiersSet(p.classifier) => - p + val retained = + overrideClassifiers match { + case None => + project.publications.collect { + case (conf, p) + if conf == "*" || + conf == dependency.configuration || + project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) => + p + } + case Some(classifiers) => + val classifiersSet = classifiers.toSet + project.publications.collect { + case (_, p) if classifiersSet(p.classifier) => + p + } } + + val retainedWithUrl = retained.flatMap { p => + substitute(variables( + dependency.module, + dependency.version, + p.`type`, + p.name, + p.ext + )).toList.map(p -> _) + } + + retainedWithUrl.map { case (p, url) => + var artifact = Artifact( + url, + Map.empty, + Map.empty, + p.attributes, + changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable + ) + + if (withChecksums) + artifact = artifact.withDefaultChecksums + if (withSignatures) + artifact = artifact.withDefaultSignature + + artifact + } } - - val retainedWithUrl = retained.flatMap { p => - substitute(variables( - dependency.module, - dependency.version, - p.`type`, - p.name, - p.ext - )).toList.map(p -> _) } - - retainedWithUrl.map { case (p, url) => - var artifact = Artifact( - url, - Map.empty, - Map.empty, - p.attributes, - changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable - ) - - if (withChecksums) - artifact = artifact.withDefaultChecksums - if (withSignatures) - artifact = artifact.withDefaultSignature - - artifact - } - } - } + else + Artifact.Source.empty def find[F[_]]( diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index c6b2109c6..8cce5ddff 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -81,7 +81,8 @@ object CoursierPlugin extends AutoPlugin { new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString + "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]", withChecksums = false, - withSignatures = false + withSignatures = false, + withArtifacts = false ) val interProjectRepo = InterProjectRepository(projects) From 24f8c430a4d761168f30ebf8206d3d7413a22780 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:39 +0100 Subject: [PATCH 32/80] Don't check MD5 sums --- plugin/src/main/scala/coursier/CoursierPlugin.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 8cce5ddff..a082e3505 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -279,7 +279,7 @@ object CoursierPlugin extends AutoPlugin { override lazy val projectSettings = Seq( coursierParallelDownloads := 6, coursierMaxIterations := 50, - coursierChecksums := Seq(Some("SHA-1"), Some("MD5"), None), + coursierChecksums := Seq(Some("SHA-1"), None), coursierCachePolicy := CachePolicy.FetchMissing, coursierVerbosity := 1, coursierResolvers <<= Tasks.coursierResolversTask, From 9bab2d9178bc9ddbdc212c952b1519f268514292 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:39 +0100 Subject: [PATCH 33/80] Move thing around, add support for updateSbtClassifiers --- .../main/scala/coursier/CoursierPlugin.scala | 165 +++++++++--------- plugin/src/main/scala/coursier/Keys.scala | 4 +- plugin/src/main/scala/coursier/ToSbt.scala | 93 ++++++++++ 3 files changed, 177 insertions(+), 85 deletions(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index a082e3505..3d85bbfc6 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -12,6 +12,12 @@ import scalaz.concurrent.Task object CoursierPlugin extends AutoPlugin { + private def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = + map.groupBy { case (k, _) => k }.map { + case (k, l) => + k -> l.map { case (_, v) => v } + } + override def trigger = allRequirements override def requires = sbt.plugins.IvyPlugin @@ -28,6 +34,7 @@ object CoursierPlugin extends AutoPlugin { val coursierCache = Keys.coursierCache val coursierProject = Keys.coursierProject val coursierProjects = Keys.coursierProjects + val coursierSbtClassifiersModule = Keys.coursierSbtClassifiersModule } import autoImport._ @@ -45,13 +52,28 @@ object CoursierPlugin extends AutoPlugin { } - private def updateTask(withClassifiers: Boolean) = Def.task { + private def updateTask(withClassifiers: Boolean, sbtClassifiers: Boolean = false) = Def.task { // let's update only one module at once, for a better output // Downloads are already parallel, no need to parallelize further anyway synchronized { - val (currentProject, _) = coursierProject.value + lazy val cm = coursierSbtClassifiersModule.value + + val currentProject = + if (sbtClassifiers) { + FromSbt.project( + cm.id, + cm.modules, + cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap, + scalaVersion.value, + scalaBinaryVersion.value + ) + } else { + val (p, _) = coursierProject.value + p + } + val projects = coursierProjects.value val parallelDownloads = coursierParallelDownloads.value @@ -157,6 +179,7 @@ object CoursierPlugin extends AutoPlugin { } val errors = res.errors + if (errors.nonEmpty) { println(s"\n${errors.size} error(s):") for ((dep, errs) <- errors) { @@ -166,7 +189,12 @@ object CoursierPlugin extends AutoPlugin { val classifiers = if (withClassifiers) - Some(transitiveClassifiers.value) + Some { + if (sbtClassifiers) + cm.classifiers + else + transitiveClassifiers.value + } else None @@ -176,102 +204,69 @@ object CoursierPlugin extends AutoPlugin { case Some(cl) => res.classifiersArtifacts(cl) } - val trDepsWithArtifactsTasks = allArtifacts - .toVector - .map { a => - files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) - } + val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => + files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) + } if (verbosity >= 0) errPrintln(s"Fetching artifacts") - // rename - val trDepsWithArtifacts = Task.gatherUnordered(trDepsWithArtifactsTasks).attemptRun match { + + val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { case -\/(ex) => throw new Exception(s"Error while downloading / verifying artifacts", ex) - case \/-(l) => l.toMap + case \/-(l) => + l.toMap } + if (verbosity >= 0) errPrintln(s"Fetching artifacts: done") - val configs = ivyConfigurations.value.map(c => c.name -> c.extendsConfigs.map(_.name)).toMap - def allExtends(c: String) = { - // possibly bad complexity - def helper(current: Set[String]): Set[String] = { - val newSet = current ++ current.flatMap(configs.getOrElse(_, Nil)) - if ((newSet -- current).nonEmpty) - helper(newSet) - else - newSet + val configs = { + val configs0 = ivyConfigurations.value.map { config => + config.name -> config.extendsConfigs.map(_.name) + }.toMap + + def allExtends(c: String) = { + // possibly bad complexity + def helper(current: Set[String]): Set[String] = { + val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) + if ((newSet -- current).nonEmpty) + helper(newSet) + else + newSet + } + + helper(Set(c)) } - helper(Set(c)) + configs0.map { + case (config, _) => + config -> allExtends(config) + } } - val depsByConfig = currentProject - .dependencies - .groupBy { case (c, _) => c } - .map { case (c, l) => - c -> l.map { case (_, d) => d } + def artifactFileOpt(artifact: Artifact) = { + val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded")) + + fileOrError match { + case \/-(file) => + if (file.toString.contains("file:/")) + throw new Exception(s"Wrong path: $file") + Some(file) + case -\/(err) => + errPrintln(s"${artifact.url}: $err") + None } + } - val sbtModuleReportsPerScope = configs.map { case (c, _) => c -> { - val a = allExtends(c).flatMap(depsByConfig.getOrElse(_, Nil)) - val partialRes = res.part(a) - val depArtifacts = - classifiers match { - case None => partialRes.dependencyArtifacts - case Some(cl) => partialRes.dependencyClassifiersArtifacts(cl) - } + val depsByConfig = grouped(currentProject.dependencies) - depArtifacts - .groupBy { case (dep, _) => dep } - .map { case (dep, l) => dep -> l.map { case (_, a) => a } } - .map { case (dep, artifacts) => - val fe = artifacts.map { a => - a -> trDepsWithArtifacts.getOrElse(a, -\/("Not downloaded")) - } - new ModuleReport( - ModuleID(dep.module.organization, dep.module.name, dep.version, configurations = Some(dep.configuration)), - fe.collect { case (artifact, \/-(file)) => - if (file.toString.contains("file:/")) - throw new Exception(s"Wrong path: $file") - ToSbt.artifact(dep.module, artifact) -> file - }, - fe.collect { case (artifact, -\/(e)) => - errPrintln(s"${artifact.url}: $e") - ToSbt.artifact(dep.module, artifact) - }, - None, - None, - None, - None, - false, - None, - None, - None, - None, - Map.empty, - None, - None, - Nil, - Nil, - Nil - ) - } - }} - - new UpdateReport( - null, - sbtModuleReportsPerScope.toVector.map { case (c, r) => - new ConfigurationReport( - c, - r.toVector, - Nil, - Nil - ) - }, - new UpdateStats(-1L, -1L, -1L, cached = false), - Map.empty + ToSbt.updateReport( + depsByConfig, + res, + configs, + classifiers, + artifactFileOpt ) } } @@ -286,8 +281,10 @@ object CoursierPlugin extends AutoPlugin { coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), update <<= updateTask(withClassifiers = false), updateClassifiers <<= updateTask(withClassifiers = true), + updateSbtClassifiers in Defaults.TaskGlobal <<= updateTask(withClassifiers = true, sbtClassifiers = true), coursierProject <<= Tasks.coursierProjectTask, - coursierProjects <<= Tasks.coursierProjectsTask + coursierProjects <<= Tasks.coursierProjectsTask, + coursierSbtClassifiersModule <<= classifiersModule in updateSbtClassifiers ) } diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala index ba66e8178..506684174 100644 --- a/plugin/src/main/scala/coursier/Keys.scala +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -1,7 +1,7 @@ package coursier import java.io.File -import sbt.{ Resolver, SettingKey, TaskKey } +import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey } object Keys { val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads", "") // 6 @@ -17,4 +17,6 @@ object Keys { val coursierProject = TaskKey[(Project, Seq[(String, Seq[Artifact])])]("coursier-project", "") val coursierProjects = TaskKey[Seq[(Project, Seq[(String, Seq[Artifact])])]]("coursier-projects", "") + + val coursierSbtClassifiersModule = TaskKey[GetClassifiersModule]("coursier-sbt-classifiers-module", "") } diff --git a/plugin/src/main/scala/coursier/ToSbt.scala b/plugin/src/main/scala/coursier/ToSbt.scala index 4f003d4f4..552327ca8 100644 --- a/plugin/src/main/scala/coursier/ToSbt.scala +++ b/plugin/src/main/scala/coursier/ToSbt.scala @@ -4,6 +4,15 @@ import sbt._ object ToSbt { + def moduleId(dependency: Dependency): sbt.ModuleID = + sbt.ModuleID( + dependency.module.organization, + dependency.module.name, + dependency.version, + configurations = Some(dependency.configuration), + extraAttributes = dependency.module.attributes + ) + def artifact(module: Module, artifact: Artifact): sbt.Artifact = sbt.Artifact( module.name, @@ -15,4 +24,88 @@ object ToSbt { Map.empty ) + def moduleReport(dependency: Dependency, artifacts: Seq[(Artifact, Option[File])]): sbt.ModuleReport = + new sbt.ModuleReport( + ToSbt.moduleId(dependency), + artifacts.collect { + case (artifact, Some(file)) => + (ToSbt.artifact(dependency.module, artifact), file) + }, + artifacts.collect { + case (artifact, None) => + ToSbt.artifact(dependency.module, artifact) + }, + None, + None, + None, + None, + false, + None, + None, + None, + None, + Map.empty, + None, + None, + Nil, + Nil, + Nil + ) + + private def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = + map.groupBy { case (k, _) => k }.map { + case (k, l) => + k -> l.map { case (_, v) => v } + } + + def moduleReports( + res: Resolution, + classifiersOpt: Option[Seq[String]], + artifactFileOpt: Artifact => Option[File] + ) = { + val depArtifacts = + classifiersOpt match { + case None => res.dependencyArtifacts + case Some(cl) => res.dependencyClassifiersArtifacts(cl) + } + + val groupedDepArtifacts = grouped(depArtifacts) + + groupedDepArtifacts.map { + case (dep, artifacts) => + ToSbt.moduleReport(dep, artifacts.map(a => a -> artifactFileOpt(a))) + } + } + + def updateReport( + configDependencies: Map[String, Seq[Dependency]], + resolution: Resolution, + configs: Map[String, Set[String]], + classifiersOpt: Option[Seq[String]], + artifactFileOpt: Artifact => Option[File] + ): sbt.UpdateReport = { + + val configReports = configs.map { + case (config, extends0) => + val configDeps = extends0.flatMap(configDependencies.getOrElse(_, Nil)) + val partialRes = resolution.part(configDeps) + + val reports = ToSbt.moduleReports(partialRes, classifiersOpt, artifactFileOpt) + + new ConfigurationReport( + config, + reports.toVector, + Nil, + Nil + ) + } + + new UpdateReport( + null, + configReports.toVector, + new UpdateStats(-1L, -1L, -1L, cached = false), + Map.empty + ) + } + } From d1e502b8f3a7b9a01eda0cfef78e15e7f0cd5dad Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:39 +0100 Subject: [PATCH 34/80] Move things around --- .../main/scala/coursier/CoursierPlugin.scala | 254 +----------------- plugin/src/main/scala/coursier/Tasks.scala | 245 +++++++++++++++++ 2 files changed, 250 insertions(+), 249 deletions(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 3d85bbfc6..97ac8caff 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -1,29 +1,16 @@ package coursier -import java.io.{ File, OutputStreamWriter } +import java.io.File -import coursier.cli.TermDisplay -import coursier.ivy.IvyRepository -import sbt.{ MavenRepository => _, _ } +import sbt._ import sbt.Keys._ -import scalaz.{ -\/, \/- } -import scalaz.concurrent.Task - object CoursierPlugin extends AutoPlugin { - private def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = - map.groupBy { case (k, _) => k }.map { - case (k, l) => - k -> l.map { case (_, v) => v } - } - override def trigger = allRequirements override def requires = sbt.plugins.IvyPlugin - private def errPrintln(s: String): Unit = scala.Console.err.println(s) - object autoImport { val coursierParallelDownloads = Keys.coursierParallelDownloads val coursierMaxIterations = Keys.coursierMaxIterations @@ -40,237 +27,6 @@ object CoursierPlugin extends AutoPlugin { import autoImport._ - private val ivyProperties = Map( - "ivy.home" -> s"${sys.props("user.home")}/.ivy2" - ) ++ sys.props - - private def createLogger() = Some { - new TermDisplay( - new OutputStreamWriter(System.err), - fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty - ) - } - - - private def updateTask(withClassifiers: Boolean, sbtClassifiers: Boolean = false) = Def.task { - - // let's update only one module at once, for a better output - // Downloads are already parallel, no need to parallelize further anyway - synchronized { - - lazy val cm = coursierSbtClassifiersModule.value - - val currentProject = - if (sbtClassifiers) { - FromSbt.project( - cm.id, - cm.modules, - cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap, - scalaVersion.value, - scalaBinaryVersion.value - ) - } else { - val (p, _) = coursierProject.value - p - } - - val projects = coursierProjects.value - - val parallelDownloads = coursierParallelDownloads.value - val checksums = coursierChecksums.value - val maxIterations = coursierMaxIterations.value - val cachePolicy = coursierCachePolicy.value - val cacheDir = coursierCache.value - - val resolvers = coursierResolvers.value - - val verbosity = coursierVerbosity.value - - - val startRes = Resolution( - currentProject.dependencies.map { case (_, dep) => dep }.toSet, - filter = Some(dep => !dep.optional), - forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap - ) - - if (verbosity >= 1) { - println("InterProjectRepository") - for ((p, _) <- projects) - println(s" ${p.module}:${p.version}") - } - - val globalPluginsRepo = IvyRepository( - new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString + - "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]", - withChecksums = false, - withSignatures = false, - withArtifacts = false - ) - - val interProjectRepo = InterProjectRepository(projects) - val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) - - val files = Files( - Seq("http://" -> new File(cacheDir, "http"), "https://" -> new File(cacheDir, "https")), - () => ???, - concurrentDownloadCount = parallelDownloads - ) - - val logger = createLogger() - logger.foreach(_.init()) - val fetch = coursier.Fetch( - repositories, - files.fetch(checksums = checksums, logger = logger)(cachePolicy = CachePolicy.LocalOnly), - files.fetch(checksums = checksums, logger = logger)(cachePolicy = cachePolicy) - ) - - def depsRepr = currentProject.dependencies.map { case (config, dep) => - s"${dep.module}:${dep.version}:$config->${dep.configuration}" - }.sorted - - if (verbosity >= 0) - errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") - if (verbosity >= 1) - for (depRepr <- depsRepr) - errPrintln(s" $depRepr") - - val res = startRes - .process - .run(fetch, maxIterations) - .attemptRun - .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) - .merge - - if (!res.isDone) - throw new Exception(s"Maximum number of iteration reached!") - - if (verbosity >= 0) - errPrintln("Resolution done") - - def repr(dep: Dependency) = { - // dep.version can be an interval, whereas the one from project can't - val version = res - .projectCache - .get(dep.moduleVersion) - .map(_._2.version) - .getOrElse(dep.version) - val extra = - if (version == dep.version) "" - else s" ($version for ${dep.version})" - - ( - Seq( - dep.module.organization, - dep.module.name, - dep.attributes.`type` - ) ++ - Some(dep.attributes.classifier) - .filter(_.nonEmpty) - .toSeq ++ - Seq( - version - ) - ).mkString(":") + extra - } - - if (res.conflicts.nonEmpty) { - // Needs test - println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") - } - - val errors = res.errors - - if (errors.nonEmpty) { - println(s"\n${errors.size} error(s):") - for ((dep, errs) <- errors) { - println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") - } - } - - val classifiers = - if (withClassifiers) - Some { - if (sbtClassifiers) - cm.classifiers - else - transitiveClassifiers.value - } - else - None - - val allArtifacts = - classifiers match { - case None => res.artifacts - case Some(cl) => res.classifiersArtifacts(cl) - } - - val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) - } - - if (verbosity >= 0) - errPrintln(s"Fetching artifacts") - - val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { - case -\/(ex) => - throw new Exception(s"Error while downloading / verifying artifacts", ex) - case \/-(l) => - l.toMap - } - - if (verbosity >= 0) - errPrintln(s"Fetching artifacts: done") - - val configs = { - val configs0 = ivyConfigurations.value.map { config => - config.name -> config.extendsConfigs.map(_.name) - }.toMap - - def allExtends(c: String) = { - // possibly bad complexity - def helper(current: Set[String]): Set[String] = { - val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) - if ((newSet -- current).nonEmpty) - helper(newSet) - else - newSet - } - - helper(Set(c)) - } - - configs0.map { - case (config, _) => - config -> allExtends(config) - } - } - - def artifactFileOpt(artifact: Artifact) = { - val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded")) - - fileOrError match { - case \/-(file) => - if (file.toString.contains("file:/")) - throw new Exception(s"Wrong path: $file") - Some(file) - case -\/(err) => - errPrintln(s"${artifact.url}: $err") - None - } - } - - val depsByConfig = grouped(currentProject.dependencies) - - ToSbt.updateReport( - depsByConfig, - res, - configs, - classifiers, - artifactFileOpt - ) - } - } - override lazy val projectSettings = Seq( coursierParallelDownloads := 6, coursierMaxIterations := 50, @@ -279,9 +35,9 @@ object CoursierPlugin extends AutoPlugin { coursierVerbosity := 1, coursierResolvers <<= Tasks.coursierResolversTask, coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), - update <<= updateTask(withClassifiers = false), - updateClassifiers <<= updateTask(withClassifiers = true), - updateSbtClassifiers in Defaults.TaskGlobal <<= updateTask(withClassifiers = true, sbtClassifiers = true), + update <<= Tasks.updateTask(withClassifiers = false), + updateClassifiers <<= Tasks.updateTask(withClassifiers = true), + updateSbtClassifiers in Defaults.TaskGlobal <<= Tasks.updateTask(withClassifiers = true, sbtClassifiers = true), coursierProject <<= Tasks.coursierProjectTask, coursierProjects <<= Tasks.coursierProjectsTask, coursierSbtClassifiersModule <<= classifiersModule in updateSbtClassifiers diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 130af086b..223909724 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -1,10 +1,17 @@ package coursier +import java.io.{OutputStreamWriter, File} + +import coursier.cli.TermDisplay +import coursier.ivy.IvyRepository import sbt.{Classpaths, Resolver, Def} import Structure._ import Keys._ import sbt.Keys._ +import scalaz.{\/-, -\/} +import scalaz.concurrent.Task + object Tasks { def coursierResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = Def.task { @@ -66,4 +73,242 @@ object Tasks { coursierProject.forAllProjects(state, projects).map(_.values.toVector) } + def updateTask(withClassifiers: Boolean, sbtClassifiers: Boolean = false) = Def.task { + + def errPrintln(s: String): Unit = scala.Console.err.println(s) + + def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = + map.groupBy { case (k, _) => k }.map { + case (k, l) => + k -> l.map { case (_, v) => v } + } + + val ivyProperties = Map( + "ivy.home" -> s"${sys.props("user.home")}/.ivy2" + ) ++ sys.props + + def createLogger() = Some { + new TermDisplay( + new OutputStreamWriter(System.err), + fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty + ) + } + + // let's update only one module at once, for a better output + // Downloads are already parallel, no need to parallelize further anyway + synchronized { + + lazy val cm = coursierSbtClassifiersModule.value + + val currentProject = + if (sbtClassifiers) + FromSbt.project( + cm.id, + cm.modules, + cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap, + scalaVersion.value, + scalaBinaryVersion.value + ) + else { + val (p, _) = coursierProject.value + p + } + + val projects = coursierProjects.value + + val parallelDownloads = coursierParallelDownloads.value + val checksums = coursierChecksums.value + val maxIterations = coursierMaxIterations.value + val cachePolicy = coursierCachePolicy.value + val cacheDir = coursierCache.value + + val resolvers = coursierResolvers.value + + val verbosity = coursierVerbosity.value + + + val startRes = Resolution( + currentProject.dependencies.map { case (_, dep) => dep }.toSet, + filter = Some(dep => !dep.optional), + forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap + ) + + if (verbosity >= 1) { + println("InterProjectRepository") + for ((p, _) <- projects) + println(s" ${p.module}:${p.version}") + } + + val globalPluginsRepo = IvyRepository( + new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString + + "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]", + withChecksums = false, + withSignatures = false, + withArtifacts = false + ) + + val interProjectRepo = InterProjectRepository(projects) + val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) + + val files = Files( + Seq("http://" -> new File(cacheDir, "http"), "https://" -> new File(cacheDir, "https")), + () => ???, + concurrentDownloadCount = parallelDownloads + ) + + val logger = createLogger() + logger.foreach(_.init()) + val fetch = coursier.Fetch( + repositories, + files.fetch(checksums = checksums, logger = logger)(cachePolicy = CachePolicy.LocalOnly), + files.fetch(checksums = checksums, logger = logger)(cachePolicy = cachePolicy) + ) + + def depsRepr = currentProject.dependencies.map { case (config, dep) => + s"${dep.module}:${dep.version}:$config->${dep.configuration}" + }.sorted + + if (verbosity >= 0) + errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") + if (verbosity >= 1) + for (depRepr <- depsRepr) + errPrintln(s" $depRepr") + + val res = startRes + .process + .run(fetch, maxIterations) + .attemptRun + .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) + .merge + + if (!res.isDone) + throw new Exception(s"Maximum number of iteration reached!") + + if (verbosity >= 0) + errPrintln("Resolution done") + + def repr(dep: Dependency) = { + // dep.version can be an interval, whereas the one from project can't + val version = res + .projectCache + .get(dep.moduleVersion) + .map(_._2.version) + .getOrElse(dep.version) + val extra = + if (version == dep.version) "" + else s" ($version for ${dep.version})" + + ( + Seq( + dep.module.organization, + dep.module.name, + dep.attributes.`type` + ) ++ + Some(dep.attributes.classifier) + .filter(_.nonEmpty) + .toSeq ++ + Seq( + version + ) + ).mkString(":") + extra + } + + if (res.conflicts.nonEmpty) { + // Needs test + println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") + } + + val errors = res.errors + + if (errors.nonEmpty) { + println(s"\n${errors.size} error(s):") + for ((dep, errs) <- errors) { + println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") + } + } + + val classifiers = + if (withClassifiers) + Some { + if (sbtClassifiers) + cm.classifiers + else + transitiveClassifiers.value + } + else + None + + val allArtifacts = + classifiers match { + case None => res.artifacts + case Some(cl) => res.classifiersArtifacts(cl) + } + + val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => + files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) + } + + if (verbosity >= 0) + errPrintln(s"Fetching artifacts") + + val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { + case -\/(ex) => + throw new Exception(s"Error while downloading / verifying artifacts", ex) + case \/-(l) => + l.toMap + } + + if (verbosity >= 0) + errPrintln(s"Fetching artifacts: done") + + val configs = { + val configs0 = ivyConfigurations.value.map { config => + config.name -> config.extendsConfigs.map(_.name) + }.toMap + + def allExtends(c: String) = { + // possibly bad complexity + def helper(current: Set[String]): Set[String] = { + val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) + if ((newSet -- current).nonEmpty) + helper(newSet) + else + newSet + } + + helper(Set(c)) + } + + configs0.map { + case (config, _) => + config -> allExtends(config) + } + } + + def artifactFileOpt(artifact: Artifact) = { + val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded")) + + fileOrError match { + case \/-(file) => + if (file.toString.contains("file:/")) + throw new Exception(s"Wrong path: $file") + Some(file) + case -\/(err) => + errPrintln(s"${artifact.url}: $err") + None + } + } + + val depsByConfig = grouped(currentProject.dependencies) + + ToSbt.updateReport( + depsByConfig, + res, + configs, + classifiers, + artifactFileOpt + ) + } + } + } From 2d61a9ea6993c9cd98112841292cfc6cbd8d3fcf Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:40 +0100 Subject: [PATCH 35/80] Remove deprecated comment --- core/shared/src/main/scala/coursier/ivy/IvyRepository.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 82a77a0d7..04582ee6e 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -143,7 +143,6 @@ case class IvyRepository( } yield acc + s } - // If attributes are added to `Module`, they should be added here // See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a // list of variables that should be supported. // Some are missing (branch, conf, originalName). From e9e034f8175d0bb5dc8f9003481787880211f9cc Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:40 +0100 Subject: [PATCH 36/80] Clean-up --- plugin/src/main/scala/coursier/Keys.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala index 506684174..609b25b9f 100644 --- a/plugin/src/main/scala/coursier/Keys.scala +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -4,10 +4,10 @@ import java.io.File import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey } object Keys { - val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads", "") // 6 - val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations", "") // 50 - val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "") //Seq(Some("SHA-1"), Some("MD5")) - val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "") // = CachePolicy.FetchMissing + val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads", "") + val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations", "") + val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "") + val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "") val coursierVerbosity = SettingKey[Int]("coursier-verbosity", "") From 790104f06a5bd68fe87947b7aefc29790dbf364a Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:40 +0100 Subject: [PATCH 37/80] Add support for extra attributes TODO Add tests --- .../src/main/scala/coursier/maven/Pom.scala | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index f6980bc11..06ece5f2f 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -168,10 +168,25 @@ object Pom { .getOrElse(Seq.empty) profiles <- xmlProfiles.toList.traverseU(profile) + extraAttrs <- properties + .collectFirst { case ("extraDependencyAttributes", s) => extraAttributes(s) } + .getOrElse(\/-(Map.empty)) + + extraAttrsMap = extraAttrs.map { + case (mod, ver) => + (mod.copy(attributes = Map.empty), ver) -> mod.attributes + }.toMap + } yield Project( projModule.copy(organization = groupId), version, - deps, + deps.map { + case (config, dep0) => + val dep = extraAttrsMap.get(dep0.moduleVersion).fold(dep0)(attrs => + dep0.copy(module = dep0.module.copy(attributes = attrs)) + ) + config -> dep + }, Map.empty, parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), depMgmts, @@ -318,4 +333,67 @@ object Pom { snapshotVersions ) } + + val extraAttributeSeparator = ":#@#:" + val extraAttributePrefix = "+" + + val extraAttributeOrg = "organisation" + val extraAttributeName = "module" + val extraAttributeVersion = "revision" + + val extraAttributeBase = Set( + extraAttributeOrg, + extraAttributeName, + extraAttributeVersion, + "branch" + ) + + def extraAttribute(s: String): String \/ (Module, String) = { + // vaguely does the same as: + // https://github.com/apache/ant-ivy/blob/2.2.0/src/java/org/apache/ivy/core/module/id/ModuleRevisionId.java#L291 + + // dropping the attributes with a value of NULL here... + + val rawParts = s.split(extraAttributeSeparator).toSeq + + val partsOrError = + if (rawParts.length % 2 == 0) { + val malformed = rawParts.filter(!_.startsWith(extraAttributePrefix)) + if (malformed.isEmpty) + \/-(rawParts.map(_.drop(extraAttributePrefix.length))) + else + -\/(s"Malformed attributes ${malformed.map("'"+_+"'").mkString(", ")} in extra attributes '$s'") + } else + -\/(s"Malformed extra attributes '$s'") + + def attrFrom(attrs: Map[String, String], name: String): String \/ String = + \/.fromEither( + attrs.get(name) + .toRight(s"$name not found in extra attributes '$s'") + ) + + for { + parts <- partsOrError + attrs = parts.grouped(2).collect { + case Seq(k, v) if v != "NULL" => + k -> v + }.toMap + org <- attrFrom(attrs, extraAttributeOrg) + name <- attrFrom(attrs, extraAttributeName) + version <- attrFrom(attrs, extraAttributeVersion) + remainingAttrs = attrs.filterKeys(!extraAttributeBase(_)) + } yield (Module(org, name, remainingAttrs.toVector.toMap), version) + } + + def extraAttributes(s: String): String \/ Seq[(Module, String)] = { + val lines = s.split('\n').toSeq.map(_.trim).filter(_.nonEmpty) + + lines.foldLeft[String \/ Seq[(Module, String)]](\/-(Vector.empty)) { + case (acc, line) => + for { + modVers <- acc + modVer <- extraAttribute(line) + } yield modVers :+ modVer + } + } } From f9a2ca5329e24298794c255f8c44a911f92977d0 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:40 +0100 Subject: [PATCH 38/80] Filter out info properties Should be done by the extraDependencyAttributes method of Module in SBT, but it looks like it's buggy here (it doesn't take the "e:" prefix into account) --- plugin/src/main/scala/coursier/FromSbt.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 06419193a..d49d8c78f 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -1,6 +1,7 @@ package coursier import coursier.ivy.IvyRepository +import sbt.mavenint.SbtPomExtraProperties import sbt.{ Resolver, CrossVersion, ModuleID } object FromSbt { @@ -31,6 +32,8 @@ object FromSbt { def attributes(attr: Map[String, String]): Map[String, String] = attr.map { case (k, v) => k.stripPrefix("e:") -> v + }.filter { case (k, _) => + !k.startsWith(SbtPomExtraProperties.POM_INFO_KEY_PREFIX) } def dependencies( From 41ba3c6c75da93044a2c3d182c1666f99bc7f3f8 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:41 +0100 Subject: [PATCH 39/80] Trim configuration names from mappings cats (as of 213aab0) does that :-| --- plugin/src/main/scala/coursier/FromSbt.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index d49d8c78f..7e77755b2 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -26,7 +26,7 @@ object FromSbt { for { from <- froms.split(',') to <- tos.split(',') - } yield (from, to) + } yield (from.trim, to.trim) } def attributes(attr: Map[String, String]): Map[String, String] = From 064f49511424ea06605c73b8550ebfa19ac3ebf0 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:41 +0100 Subject: [PATCH 40/80] Remove Files case class --- cli/src/main/scala/coursier/cli/Helper.scala | 23 +++-- files/src/main/scala/coursier/Files.scala | 95 +++++++++++--------- plugin/src/main/scala/coursier/Tasks.scala | 19 ++-- 3 files changed, 76 insertions(+), 61 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 8ec1f43ef..23d5cdf12 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -2,11 +2,12 @@ package coursier package cli import java.io.{ OutputStreamWriter, File } +import java.util.concurrent.Executors import coursier.ivy.IvyRepository import scalaz.{ \/-, -\/ } -import scalaz.concurrent.Task +import scalaz.concurrent.{ Task, Strategy } object Helper { def fileRepr(f: File) = f.toString @@ -57,16 +58,14 @@ class Helper( sys.exit(255) } - val files = - Files( - Seq( - "http://" -> new File(new File(cacheOptions.cache), "http"), - "https://" -> new File(new File(cacheOptions.cache), "https") - ), - () => ???, - concurrentDownloadCount = parallel + val caches = + Seq( + "http://" -> new File(new File(cacheOptions.cache), "http"), + "https://" -> new File(new File(cacheOptions.cache), "https") ) + val pool = Executors.newFixedThreadPool(parallel, Strategy.DefaultDaemonThreadFactory) + val central = MavenRepository("https://repo1.maven.org/maven2/") val ivy2Local = MavenRepository( new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, @@ -218,7 +217,7 @@ class Helper( logger.foreach(_.init()) val fetchs = cachePolicies.map(p => - files.fetch(logger = logger)(cachePolicy = p) + Files.fetch(caches, p, logger = logger, pool = pool) ) val fetchQuiet = coursier.Fetch( repositories, @@ -345,8 +344,8 @@ class Helper( None logger.foreach(_.init()) val tasks = artifacts.map(artifact => - (files.file(artifact, logger = logger)(cachePolicy = cachePolicies.head) /: cachePolicies.tail)( - _ orElse files.file(artifact, logger = logger)(_) + (Files.file(artifact, caches, cachePolicies.head, logger = logger, pool = pool) /: cachePolicies.tail)( + _ orElse Files.file(artifact, caches, _, logger = logger, pool = pool) ).run.map(artifact.->) ) def printTask = Task { diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index d6e5c9eb8..1d9f2f467 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -11,34 +11,28 @@ import scalaz.concurrent.{ Task, Strategy } import java.io.{ Serializable => _, _ } -case class Files( - cache: Seq[(String, File)], - tmp: () => File, - concurrentDownloadCount: Int = Files.defaultConcurrentDownloadCount -) { +object Files { - import Files.urlLocks - - lazy val defaultPool = - Executors.newFixedThreadPool(concurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) - - def withLocal(artifact: Artifact): Artifact = { + def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = { def local(url: String) = if (url.startsWith("file:///")) url.stripPrefix("file://") else if (url.startsWith("file:/")) url.stripPrefix("file:") - else - cache.find { case (base, _) => url.startsWith(base) } match { - case None => - // FIXME Means we were handed an artifact from repositories other than the known ones - println(cache.mkString("\n")) - println(url) - ??? - case Some((base, cacheDir)) => + else { + val localPathOpt = cache.collectFirst { + case (base, cacheDir) if url.startsWith(base) => cacheDir + "/" + url.stripPrefix(base) } + localPathOpt.getOrElse { + // FIXME Means we were handed an artifact from repositories other than the known ones + println(cache.mkString("\n")) + println(url) + ??? + } + } + if (artifact.extra.contains("local")) artifact else @@ -56,13 +50,16 @@ case class Files( def download( artifact: Artifact, + cache: Seq[(String, File)], checksums: Set[String], - logger: Option[Files.Logger] = None - )(implicit cachePolicy: CachePolicy, - pool: ExecutorService = defaultPool + pool: ExecutorService, + logger: Option[Files.Logger] = None ): Task[Seq[((File, String), FileError \/ Unit)]] = { - val artifact0 = withLocal(artifact) + + implicit val pool0 = pool + + val artifact0 = withLocal(artifact, cache) .extra .getOrElse("local", artifact) @@ -271,11 +268,14 @@ case class Files( def validateChecksum( artifact: Artifact, - sumType: String - )(implicit - pool: ExecutorService = defaultPool + sumType: String, + cache: Seq[(String, File)], + pool: ExecutorService ): EitherT[Task, FileError, Unit] = { - val artifact0 = withLocal(artifact) + + implicit val pool0 = pool + + val artifact0 = withLocal(artifact, cache) .extra .getOrElse("local", artifact) @@ -330,18 +330,24 @@ case class Files( def file( artifact: Artifact, - checksums: Seq[Option[String]] = Seq(Some("SHA-1")), - logger: Option[Files.Logger] = None - )(implicit + cache: Seq[(String, File)], cachePolicy: CachePolicy, - pool: ExecutorService = defaultPool + checksums: Seq[Option[String]] = Seq(Some("SHA-1")), + logger: Option[Files.Logger] = None, + pool: ExecutorService = Files.defaultPool ): EitherT[Task, FileError, File] = { + + implicit val pool0 = pool + val checksums0 = if (checksums.isEmpty) Seq(None) else checksums val res = EitherT { download( artifact, + cache, checksums = checksums0.collect { case Some(c) => c }.toSet, + cachePolicy, + pool, logger = logger ).map { results => val checksum = checksums0.find { @@ -370,28 +376,31 @@ case class Files( res.flatMap { case (f, None) => EitherT(Task.now[FileError \/ File](\/-(f))) case (f, Some(c)) => - validateChecksum(artifact, c).map(_ => f) + validateChecksum(artifact, c, cache, pool).map(_ => f) } } def fetch( - checksums: Seq[Option[String]] = Seq(Some("SHA-1")), - logger: Option[Files.Logger] = None - )(implicit + cache: Seq[(String, File)], cachePolicy: CachePolicy, - pool: ExecutorService = defaultPool + checksums: Seq[Option[String]] = Seq(Some("SHA-1")), + logger: Option[Files.Logger] = None, + pool: ExecutorService = Files.defaultPool ): Fetch.Content[Task] = { artifact => - file(artifact, checksums = checksums, logger = logger)(cachePolicy).leftMap(_.message).map { f => + file( + artifact, + cache, + cachePolicy, + checksums = checksums, + logger = logger, + pool = pool + ).leftMap(_.message).map { f => // FIXME Catch error here? scala.io.Source.fromFile(f)("UTF-8").mkString } } -} - -object Files { - lazy val ivy2Local = MavenRepository( new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, ivyLike = true @@ -399,6 +408,10 @@ object Files { val defaultConcurrentDownloadCount = 6 + lazy val defaultPool = + Executors.newFixedThreadPool(defaultConcurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) + + private val urlLocks = new ConcurrentHashMap[String, Object] trait Logger { diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 223909724..3854919b1 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -1,6 +1,7 @@ package coursier import java.io.{OutputStreamWriter, File} +import java.util.concurrent.Executors import coursier.cli.TermDisplay import coursier.ivy.IvyRepository @@ -10,7 +11,7 @@ import Keys._ import sbt.Keys._ import scalaz.{\/-, -\/} -import scalaz.concurrent.Task +import scalaz.concurrent.{ Task, Strategy } object Tasks { @@ -150,18 +151,19 @@ object Tasks { val interProjectRepo = InterProjectRepository(projects) val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) - val files = Files( - Seq("http://" -> new File(cacheDir, "http"), "https://" -> new File(cacheDir, "https")), - () => ???, - concurrentDownloadCount = parallelDownloads + val caches = Seq( + "http://" -> new File(cacheDir, "http"), + "https://" -> new File(cacheDir, "https") ) + val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) + val logger = createLogger() logger.foreach(_.init()) val fetch = coursier.Fetch( repositories, - files.fetch(checksums = checksums, logger = logger)(cachePolicy = CachePolicy.LocalOnly), - files.fetch(checksums = checksums, logger = logger)(cachePolicy = cachePolicy) + Files.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = logger, pool = pool), + Files.fetch(caches, cachePolicy, checksums = checksums, logger = logger, pool = pool) ) def depsRepr = currentProject.dependencies.map { case (config, dep) => @@ -225,6 +227,7 @@ object Tasks { for ((dep, errs) <- errors) { println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") } + throw new Exception(s"Encountered ${errors.length} error(s)") } val classifiers = @@ -245,7 +248,7 @@ object Tasks { } val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - files.file(a, checksums = checksums, logger = logger)(cachePolicy = cachePolicy).run.map((a, _)) + Files.file(a, caches, cachePolicy, checksums = checksums, logger = logger, pool = pool).run.map((a, _)) } if (verbosity >= 0) From 9f26ed05b2e17cda6bde36694c9a9dbe0117e9e2 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:41 +0100 Subject: [PATCH 41/80] Rename Files to Cache --- cli/src/main/scala/coursier/cli/Coursier.scala | 2 +- cli/src/main/scala/coursier/cli/Helper.scala | 6 +++--- .../main/scala/coursier/cli/TermDisplay.scala | 4 ++-- .../coursier/{Files.scala => Cache.scala} | 18 +++++++++--------- plugin/src/main/scala/coursier/Tasks.scala | 6 +++--- .../scala/coursier/test/IvyLocalTests.scala | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) rename files/src/main/scala/coursier/{Files.scala => Cache.scala} (97%) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 1fcbbcf8f..0f9a6e745 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -234,7 +234,7 @@ case class Bootstrap( val bootstrapJar = Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { - case Some(is) => Files.readFullySync(is) + case Some(is) => Cache.readFullySync(is) case None => Console.err.println(s"Error: bootstrap JAR not found") sys.exit(1) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 23d5cdf12..1726d8b7c 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -217,7 +217,7 @@ class Helper( logger.foreach(_.init()) val fetchs = cachePolicies.map(p => - Files.fetch(caches, p, logger = logger, pool = pool) + Cache.fetch(caches, p, logger = logger, pool = pool) ) val fetchQuiet = coursier.Fetch( repositories, @@ -344,8 +344,8 @@ class Helper( None logger.foreach(_.init()) val tasks = artifacts.map(artifact => - (Files.file(artifact, caches, cachePolicies.head, logger = logger, pool = pool) /: cachePolicies.tail)( - _ orElse Files.file(artifact, caches, _, logger = logger, pool = pool) + (Cache.file(artifact, caches, cachePolicies.head, logger = logger, pool = pool) /: cachePolicies.tail)( + _ orElse Cache.file(artifact, caches, _, logger = logger, pool = pool) ).run.map(artifact.->) ) def printTask = Task { diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cli/src/main/scala/coursier/cli/TermDisplay.scala index e4c7358b9..96f1d2695 100644 --- a/cli/src/main/scala/coursier/cli/TermDisplay.scala +++ b/cli/src/main/scala/coursier/cli/TermDisplay.scala @@ -5,7 +5,7 @@ import java.util.concurrent._ import ammonite.terminal.{ TTY, Ansi } -import coursier.Files.Logger +import coursier.Cache import scala.annotation.tailrec import scala.collection.mutable.ArrayBuffer @@ -13,7 +13,7 @@ import scala.collection.mutable.ArrayBuffer class TermDisplay( out: Writer, var fallbackMode: Boolean = false -) extends Logger { +) extends Cache.Logger { private val ansi = new Ansi(out) private var width = 80 diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Cache.scala similarity index 97% rename from files/src/main/scala/coursier/Files.scala rename to files/src/main/scala/coursier/Cache.scala index 1d9f2f467..4f30fd7e2 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Cache.scala @@ -11,7 +11,7 @@ import scalaz.concurrent.{ Task, Strategy } import java.io.{ Serializable => _, _ } -object Files { +object Cache { def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = { def local(url: String) = @@ -54,7 +54,7 @@ object Files { checksums: Set[String], cachePolicy: CachePolicy, pool: ExecutorService, - logger: Option[Files.Logger] = None + logger: Option[Logger] = None ): Task[Seq[((File, String), FileError \/ Unit)]] = { implicit val pool0 = pool @@ -159,7 +159,7 @@ object Files { for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) logger.foreach(_.downloadLength(url, len)) - val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize) + val in = new BufferedInputStream(conn.getInputStream, bufferSize) val result = try { @@ -172,7 +172,7 @@ object Files { if (lock == null) -\/(FileError.Locked(file)) else { - val b = Array.fill[Byte](Files.bufferSize)(0) + val b = Array.fill[Byte](bufferSize)(0) @tailrec def helper(count: Long): Unit = { @@ -300,7 +300,7 @@ object Files { if (lock == null) -\/(FileError.Locked(f)) else { - Files.withContent(is, md.update(_, 0, _)) + withContent(is, md.update(_, 0, _)) \/-(()) } } @@ -333,8 +333,8 @@ object Files { cache: Seq[(String, File)], cachePolicy: CachePolicy, checksums: Seq[Option[String]] = Seq(Some("SHA-1")), - logger: Option[Files.Logger] = None, - pool: ExecutorService = Files.defaultPool + logger: Option[Logger] = None, + pool: ExecutorService = defaultPool ): EitherT[Task, FileError, File] = { implicit val pool0 = pool @@ -384,8 +384,8 @@ object Files { cache: Seq[(String, File)], cachePolicy: CachePolicy, checksums: Seq[Option[String]] = Seq(Some("SHA-1")), - logger: Option[Files.Logger] = None, - pool: ExecutorService = Files.defaultPool + logger: Option[Logger] = None, + pool: ExecutorService = defaultPool ): Fetch.Content[Task] = { artifact => file( diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 3854919b1..8de0a9354 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -162,8 +162,8 @@ object Tasks { logger.foreach(_.init()) val fetch = coursier.Fetch( repositories, - Files.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = logger, pool = pool), - Files.fetch(caches, cachePolicy, checksums = checksums, logger = logger, pool = pool) + Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = logger, pool = pool), + Cache.fetch(caches, cachePolicy, checksums = checksums, logger = logger, pool = pool) ) def depsRepr = currentProject.dependencies.map { case (config, dep) => @@ -248,7 +248,7 @@ object Tasks { } val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - Files.file(a, caches, cachePolicy, checksums = checksums, logger = logger, pool = pool).run.map((a, _)) + Cache.file(a, caches, cachePolicy, checksums = checksums, logger = logger, pool = pool).run.map((a, _)) } if (verbosity >= 0) diff --git a/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala index ac0c31865..05aa0a712 100644 --- a/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.{ Module, Files } +import coursier.{ Module, Cache } import utest._ object IvyLocalTests extends TestSuite { @@ -10,7 +10,7 @@ object IvyLocalTests extends TestSuite { // Assume this module (and the sub-projects it depends on) is published locally CentralTests.resolutionCheck( Module("com.github.alexarchambault", "coursier_2.11"), "0.1.0-SNAPSHOT", - Some(Files.ivy2Local)) + Some(Cache.ivy2Local)) } } From c48c50f95931fcfcdf0986bdc9313ff51b8429a2 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:41 +0100 Subject: [PATCH 42/80] Rename files project to cache --- build.sbt | 14 +++++++------- .../src/main/scala/coursier/Cache.scala | 0 .../src/main/scala/coursier/CachePolicy.scala | 0 .../src/main/scala/coursier/Platform.scala | 0 4 files changed, 7 insertions(+), 7 deletions(-) rename {files => cache}/src/main/scala/coursier/Cache.scala (100%) rename {files => cache}/src/main/scala/coursier/CachePolicy.scala (100%) rename {files => cache}/src/main/scala/coursier/Platform.scala (100%) diff --git a/build.sbt b/build.sbt index 353a10f92..99e0aafeb 100644 --- a/build.sbt +++ b/build.sbt @@ -128,15 +128,15 @@ lazy val tests = crossProject scalaJSStage in Global := FastOptStage ) -lazy val testsJvm = tests.jvm.dependsOn(files % "test") +lazy val testsJvm = tests.jvm.dependsOn(cache % "test") lazy val testsJs = tests.js.dependsOn(`fetch-js` % "test") -lazy val files = project +lazy val cache = project .dependsOn(coreJvm) .settings(commonSettings) .settings(publishingSettings) .settings( - name := "coursier-files", + name := "coursier-cache", libraryDependencies ++= Seq( "org.scalaz" %% "scalaz-concurrent" % "7.1.2" ) @@ -160,7 +160,7 @@ lazy val bootstrap = project ) lazy val cli = project - .dependsOn(coreJvm, files) + .dependsOn(coreJvm, cache) .settings(commonSettings) .settings(publishingSettings) .settings(packAutoSettings) @@ -205,7 +205,7 @@ lazy val web = project ) lazy val doc = project - .dependsOn(coreJvm, files) + .dependsOn(coreJvm, cache) .settings(commonSettings) .settings(noPublishSettings) .settings(tutSettings) @@ -216,7 +216,7 @@ lazy val doc = project // Don't try to compile that if you're not in 2.10 lazy val plugin = project - .dependsOn(coreJvm, files, cli) + .dependsOn(coreJvm, cache, cli) .settings(baseCommonSettings) .settings( name := "coursier-sbt-plugin", @@ -224,7 +224,7 @@ lazy val plugin = project ) lazy val `coursier` = project.in(file(".")) - .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web, doc) + .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, cache, bootstrap, cli, web, doc) .settings(commonSettings) .settings(noPublishSettings) .settings(releaseSettings) diff --git a/files/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala similarity index 100% rename from files/src/main/scala/coursier/Cache.scala rename to cache/src/main/scala/coursier/Cache.scala diff --git a/files/src/main/scala/coursier/CachePolicy.scala b/cache/src/main/scala/coursier/CachePolicy.scala similarity index 100% rename from files/src/main/scala/coursier/CachePolicy.scala rename to cache/src/main/scala/coursier/CachePolicy.scala diff --git a/files/src/main/scala/coursier/Platform.scala b/cache/src/main/scala/coursier/Platform.scala similarity index 100% rename from files/src/main/scala/coursier/Platform.scala rename to cache/src/main/scala/coursier/Platform.scala From dbe2b801ac36098c7d936f35887787dd91456689 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:42 +0100 Subject: [PATCH 43/80] Better handling of repositories in updateSbtClassifiers task --- plugin/src/main/scala/coursier/CoursierPlugin.scala | 2 ++ plugin/src/main/scala/coursier/Keys.scala | 1 + plugin/src/main/scala/coursier/Tasks.scala | 6 +++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 97ac8caff..7375e5569 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -18,6 +18,7 @@ object CoursierPlugin extends AutoPlugin { val coursierCachePolicy = Keys.coursierCachePolicy val coursierVerbosity = Keys.coursierVerbosity val coursierResolvers = Keys.coursierResolvers + val coursierSbtResolvers = Keys.coursierSbtResolvers val coursierCache = Keys.coursierCache val coursierProject = Keys.coursierProject val coursierProjects = Keys.coursierProjects @@ -34,6 +35,7 @@ object CoursierPlugin extends AutoPlugin { coursierCachePolicy := CachePolicy.FetchMissing, coursierVerbosity := 1, coursierResolvers <<= Tasks.coursierResolversTask, + coursierSbtResolvers <<= externalResolvers in updateSbtClassifiers, coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), update <<= Tasks.updateTask(withClassifiers = false), updateClassifiers <<= Tasks.updateTask(withClassifiers = true), diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala index 609b25b9f..7ce7bd847 100644 --- a/plugin/src/main/scala/coursier/Keys.scala +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -12,6 +12,7 @@ object Keys { val coursierVerbosity = SettingKey[Int]("coursier-verbosity", "") val coursierResolvers = TaskKey[Seq[Resolver]]("coursier-resolvers", "") + val coursierSbtResolvers = TaskKey[Seq[Resolver]]("coursier-sbt-resolvers", "") val coursierCache = SettingKey[File]("coursier-cache", "") diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 8de0a9354..883696910 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -123,7 +123,11 @@ object Tasks { val cachePolicy = coursierCachePolicy.value val cacheDir = coursierCache.value - val resolvers = coursierResolvers.value + val resolvers = + if (sbtClassifiers) + coursierSbtResolvers.value + else + coursierResolvers.value val verbosity = coursierVerbosity.value From 776bcb893f00caa2cb5a406d30710eec57c934f4 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:42 +0100 Subject: [PATCH 44/80] Add classifier variable in Ivy patterns --- .../src/main/scala/coursier/ivy/IvyRepository.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 04582ee6e..fb14f9738 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -151,7 +151,8 @@ case class IvyRepository( version: String, `type`: String, artifact: String, - ext: String + ext: String, + classifierOpt: Option[String] ) = Map( "organization" -> module.organization, @@ -162,7 +163,7 @@ case class IvyRepository( "type" -> `type`, "artifact" -> artifact, "ext" -> ext - ) ++ module.attributes + ) ++ module.attributes ++ classifierOpt.map("classifier" -> _).toSeq val source: Artifact.Source = @@ -198,7 +199,8 @@ case class IvyRepository( dependency.version, p.`type`, p.name, - p.ext + p.ext, + Some(p.classifier).filter(_.nonEmpty) )).toList.map(p -> _) } @@ -235,7 +237,7 @@ case class IvyRepository( val eitherArtifact: String \/ Artifact = for { url <- substitute( - variables(module, version, "ivy", "ivy", "xml") + variables(module, version, "ivy", "ivy", "xml", None) ) } yield { var artifact = Artifact( From 02a0ed5866c9e426db15888bdfdcb54fd6296842 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:42 +0100 Subject: [PATCH 45/80] Disable checksum verifying for artifacts by default in SBT plugin --- plugin/src/main/scala/coursier/CoursierPlugin.scala | 2 ++ plugin/src/main/scala/coursier/Keys.scala | 1 + plugin/src/main/scala/coursier/Tasks.scala | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index 7375e5569..cd9e4ebc3 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -15,6 +15,7 @@ object CoursierPlugin extends AutoPlugin { val coursierParallelDownloads = Keys.coursierParallelDownloads val coursierMaxIterations = Keys.coursierMaxIterations val coursierChecksums = Keys.coursierChecksums + val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums val coursierCachePolicy = Keys.coursierCachePolicy val coursierVerbosity = Keys.coursierVerbosity val coursierResolvers = Keys.coursierResolvers @@ -32,6 +33,7 @@ object CoursierPlugin extends AutoPlugin { coursierParallelDownloads := 6, coursierMaxIterations := 50, coursierChecksums := Seq(Some("SHA-1"), None), + coursierArtifactsChecksums := Seq(None), coursierCachePolicy := CachePolicy.FetchMissing, coursierVerbosity := 1, coursierResolvers <<= Tasks.coursierResolversTask, diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala index 7ce7bd847..6fa28d804 100644 --- a/plugin/src/main/scala/coursier/Keys.scala +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -7,6 +7,7 @@ object Keys { val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads", "") val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations", "") val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "") + val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums", "") val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "") val coursierVerbosity = SettingKey[Int]("coursier-verbosity", "") diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 883696910..8f44b21f6 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -119,6 +119,7 @@ object Tasks { val parallelDownloads = coursierParallelDownloads.value val checksums = coursierChecksums.value + val artifactsChecksums = coursierArtifactsChecksums.value val maxIterations = coursierMaxIterations.value val cachePolicy = coursierCachePolicy.value val cacheDir = coursierCache.value @@ -252,7 +253,7 @@ object Tasks { } val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - Cache.file(a, caches, cachePolicy, checksums = checksums, logger = logger, pool = pool).run.map((a, _)) + Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = logger, pool = pool).run.map((a, _)) } if (verbosity >= 0) From d47e6b074ac06386ac5fea2096c3276f40f713ef Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:42 +0100 Subject: [PATCH 46/80] Less redundant dependencies in output --- plugin/src/main/scala/coursier/Tasks.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 8f44b21f6..17d22624c 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -178,7 +178,7 @@ object Tasks { if (verbosity >= 0) errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") if (verbosity >= 1) - for (depRepr <- depsRepr) + for (depRepr <- depsRepr.distinct) errPrintln(s" $depRepr") val res = startRes From dd4dbb41f9b3e577b20174075270f44e96da8812 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:43 +0100 Subject: [PATCH 47/80] Better names / comments --- cache/src/main/scala/coursier/Cache.scala | 2 +- .../main/scala/coursier/core/Resolution.scala | 2 +- plugin/src/main/scala/coursier/FromSbt.scala | 2 +- .../coursier/InterProjectRepository.scala | 12 +++--- plugin/src/main/scala/coursier/Tasks.scala | 39 +++++++++---------- plugin/src/main/scala/coursier/ToSbt.scala | 4 +- 6 files changed, 30 insertions(+), 31 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 4f30fd7e2..5b7ccad6c 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -77,7 +77,7 @@ object Cache { // Dummy user-agent instead of the default "Java/...", // so that we are not returned incomplete/erroneous metadata // (Maven 2 compatibility? - happens for snapshot versioning metadata, - // this is SO FUCKING CRAZY) + // this is SO FSCKING CRAZY) conn.setRequestProperty("User-Agent", "") conn } diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index ec38adc46..ea035dd40 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -786,7 +786,7 @@ case class Resolution( .toSeq } yield (dep, err) - def part(dependencies: Set[Dependency]): Resolution = { + def subset(dependencies: Set[Dependency]): Resolution = { val (_, _, finalVersions) = nextDependenciesAndConflicts def updateVersion(dep: Dependency): Dependency = diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 7e77755b2..b2e22ba00 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -80,7 +80,7 @@ object FromSbt { scalaBinaryVersion: String ): Project = { - // FIXME Ignored for now + // FIXME Ignored for now - easy to support though // val sbtDepOverrides = dependencyOverrides.value // val sbtExclusions = excludeDependencies.value diff --git a/plugin/src/main/scala/coursier/InterProjectRepository.scala b/plugin/src/main/scala/coursier/InterProjectRepository.scala index aa833f4f6..3c87631e5 100644 --- a/plugin/src/main/scala/coursier/InterProjectRepository.scala +++ b/plugin/src/main/scala/coursier/InterProjectRepository.scala @@ -23,14 +23,14 @@ case class InterProjectSource(artifacts: Map[(Module, String), Map[String, Seq[A case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artifact])])]) extends Repository { private val map = projects - .map { case (proj, a) => proj.moduleVersion -> proj } + .map { case (proj, _) => proj.moduleVersion -> proj } .toMap val source = InterProjectSource( - projects.map { case (proj, a) => - val artifacts = a.toMap - val allArtifacts = proj.allConfigurations.map { case (c, ext) => - c -> ext.toSeq.flatMap(artifacts.getOrElse(_, Nil)) + projects.map { case (proj, artifactsByConfig) => + val artifacts = artifactsByConfig.toMap + val allArtifacts = proj.allConfigurations.map { case (config, extends0) => + config -> extends0.toSeq.flatMap(artifacts.getOrElse(_, Nil)) } proj.moduleVersion -> allArtifacts }.toMap @@ -47,7 +47,7 @@ case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artif case Some(proj) => \/-((source, proj)) case None => - -\/(s"Project not found: $module:$version") + -\/("Not found") } EitherT(F.point(res)) diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 17d22624c..19ed2eab4 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -16,13 +16,13 @@ import scalaz.concurrent.{ Task, Strategy } object Tasks { def coursierResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = Def.task { - var l = externalResolvers.value + var resolvers = externalResolvers.value if (sbtPlugin.value) - l = Seq( + resolvers = Seq( sbtResolver.value, Classpaths.sbtPluginReleases - ) ++ l - l + ) ++ resolvers + resolvers } def coursierProjectTask: Def.Initialize[sbt.Task[(Project, Seq[(String, Seq[Artifact])])]] = @@ -76,6 +76,7 @@ object Tasks { def updateTask(withClassifiers: Boolean, sbtClassifiers: Boolean = false) = Def.task { + // SBT logging should be better than that most of the time... def errPrintln(s: String): Unit = scala.Console.err.println(s) def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = @@ -84,17 +85,6 @@ object Tasks { k -> l.map { case (_, v) => v } } - val ivyProperties = Map( - "ivy.home" -> s"${sys.props("user.home")}/.ivy2" - ) ++ sys.props - - def createLogger() = Some { - new TermDisplay( - new OutputStreamWriter(System.err), - fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty - ) - } - // let's update only one module at once, for a better output // Downloads are already parallel, no need to parallelize further anyway synchronized { @@ -154,6 +144,11 @@ object Tasks { ) val interProjectRepo = InterProjectRepository(projects) + + val ivyProperties = Map( + "ivy.home" -> s"${sys.props("user.home")}/.ivy2" + ) ++ sys.props + val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) val caches = Seq( @@ -163,12 +158,16 @@ object Tasks { val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) - val logger = createLogger() - logger.foreach(_.init()) + val logger = new TermDisplay( + new OutputStreamWriter(System.err), + fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty + ) + logger.init() + val fetch = coursier.Fetch( repositories, - Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = logger, pool = pool), - Cache.fetch(caches, cachePolicy, checksums = checksums, logger = logger, pool = pool) + Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(logger), pool = pool), + Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(logger), pool = pool) ) def depsRepr = currentProject.dependencies.map { case (config, dep) => @@ -253,7 +252,7 @@ object Tasks { } val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = logger, pool = pool).run.map((a, _)) + Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(logger), pool = pool).run.map((a, _)) } if (verbosity >= 0) diff --git a/plugin/src/main/scala/coursier/ToSbt.scala b/plugin/src/main/scala/coursier/ToSbt.scala index 552327ca8..21b8308c3 100644 --- a/plugin/src/main/scala/coursier/ToSbt.scala +++ b/plugin/src/main/scala/coursier/ToSbt.scala @@ -88,9 +88,9 @@ object ToSbt { val configReports = configs.map { case (config, extends0) => val configDeps = extends0.flatMap(configDependencies.getOrElse(_, Nil)) - val partialRes = resolution.part(configDeps) + val subRes = resolution.subset(configDeps) - val reports = ToSbt.moduleReports(partialRes, classifiersOpt, artifactFileOpt) + val reports = ToSbt.moduleReports(subRes, classifiersOpt, artifactFileOpt) new ConfigurationReport( config, From 63aab86d54921e9a94426146e0f2890de22b60c9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:43 +0100 Subject: [PATCH 48/80] Remove dead code --- cache/src/main/scala/coursier/Cache.scala | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 5b7ccad6c..a9623d16d 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -83,17 +83,6 @@ object Cache { } - def locally(file: File, url: String): EitherT[Task, FileError, File] = - EitherT { - Task { - if (file.exists()) { - logger.foreach(_.foundLocally(url, file)) - \/-(file) - } else - -\/(FileError.NotFound(file.toString): FileError) - } - } - def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] = EitherT { Task { @@ -235,15 +224,6 @@ object Cache { for ((f, url) <- pairs) yield { val file = new File(f) - val isRemote = url != ("file:" + f) && url != ("file://" + f) - val cachePolicy0 = - if (!isRemote) - CachePolicy.LocalOnly - else if (cachePolicy == CachePolicy.UpdateChanging && !artifact.changing) - CachePolicy.FetchMissing - else - cachePolicy - val res = cachePolicy match { case CachePolicy.LocalOnly => checkFileExists(file, url) From cedc424ecb6dc466bdea07c579dfa0ef2173ed4e Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:43 +0100 Subject: [PATCH 49/80] Cache resolutions in plugin --- plugin/src/main/scala/coursier/Tasks.scala | 349 +++++++++++---------- 1 file changed, 184 insertions(+), 165 deletions(-) diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 19ed2eab4..a441a680f 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -1,16 +1,19 @@ package coursier -import java.io.{OutputStreamWriter, File} +import java.io.{ OutputStreamWriter, File } import java.util.concurrent.Executors import coursier.cli.TermDisplay import coursier.ivy.IvyRepository -import sbt.{Classpaths, Resolver, Def} -import Structure._ -import Keys._ +import coursier.Keys._ +import coursier.Structure._ + +import sbt.{ UpdateReport, Classpaths, Resolver, Def } import sbt.Keys._ -import scalaz.{\/-, -\/} +import scala.collection.mutable + +import scalaz.{ \/-, -\/ } import scalaz.concurrent.{ Task, Strategy } object Tasks { @@ -74,6 +77,15 @@ object Tasks { coursierProject.forAllProjects(state, projects).map(_.values.toVector) } + // FIXME More things should possibly be put here too (resolvers, etc.) + private case class CacheKey( + resolution: Resolution, + withClassifiers: Boolean, + sbtClassifiers: Boolean + ) + + private val resolutionsCache = new mutable.HashMap[CacheKey, UpdateReport] + def updateTask(withClassifiers: Boolean, sbtClassifiers: Boolean = false) = Def.task { // SBT logging should be better than that most of the time... @@ -129,191 +141,198 @@ object Tasks { forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap ) - if (verbosity >= 1) { - println("InterProjectRepository") - for ((p, _) <- projects) - println(s" ${p.module}:${p.version}") - } + def report = { + if (verbosity >= 1) { + println("InterProjectRepository") + for ((p, _) <- projects) + println(s" ${p.module}:${p.version}") + } - val globalPluginsRepo = IvyRepository( - new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString + - "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]", - withChecksums = false, - withSignatures = false, - withArtifacts = false - ) + val globalPluginsRepo = IvyRepository( + new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString + + "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]", + withChecksums = false, + withSignatures = false, + withArtifacts = false + ) - val interProjectRepo = InterProjectRepository(projects) + val interProjectRepo = InterProjectRepository(projects) - val ivyProperties = Map( - "ivy.home" -> s"${sys.props("user.home")}/.ivy2" - ) ++ sys.props + val ivyProperties = Map( + "ivy.home" -> s"${sys.props("user.home")}/.ivy2" + ) ++ sys.props - val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) + val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) - val caches = Seq( - "http://" -> new File(cacheDir, "http"), - "https://" -> new File(cacheDir, "https") - ) + val caches = Seq( + "http://" -> new File(cacheDir, "http"), + "https://" -> new File(cacheDir, "https") + ) - val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) + val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) - val logger = new TermDisplay( - new OutputStreamWriter(System.err), - fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty - ) - logger.init() + val logger = new TermDisplay( + new OutputStreamWriter(System.err), + fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty + ) + logger.init() - val fetch = coursier.Fetch( - repositories, - Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(logger), pool = pool), - Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(logger), pool = pool) - ) + val fetch = coursier.Fetch( + repositories, + Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(logger), pool = pool), + Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(logger), pool = pool) + ) - def depsRepr = currentProject.dependencies.map { case (config, dep) => - s"${dep.module}:${dep.version}:$config->${dep.configuration}" - }.sorted + def depsRepr = currentProject.dependencies.map { case (config, dep) => + s"${dep.module}:${dep.version}:$config->${dep.configuration}" + }.sorted - if (verbosity >= 0) - errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") - if (verbosity >= 1) - for (depRepr <- depsRepr.distinct) - errPrintln(s" $depRepr") + if (verbosity >= 0) + errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") + if (verbosity >= 1) + for (depRepr <- depsRepr.distinct) + errPrintln(s" $depRepr") - val res = startRes - .process - .run(fetch, maxIterations) - .attemptRun - .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) - .merge + val res = startRes + .process + .run(fetch, maxIterations) + .attemptRun + .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) + .merge - if (!res.isDone) - throw new Exception(s"Maximum number of iteration reached!") + if (!res.isDone) + throw new Exception(s"Maximum number of iteration reached!") - if (verbosity >= 0) - errPrintln("Resolution done") + if (verbosity >= 0) + errPrintln("Resolution done") - def repr(dep: Dependency) = { - // dep.version can be an interval, whereas the one from project can't - val version = res - .projectCache - .get(dep.moduleVersion) - .map(_._2.version) - .getOrElse(dep.version) - val extra = - if (version == dep.version) "" - else s" ($version for ${dep.version})" + def repr(dep: Dependency) = { + // dep.version can be an interval, whereas the one from project can't + val version = res + .projectCache + .get(dep.moduleVersion) + .map(_._2.version) + .getOrElse(dep.version) + val extra = + if (version == dep.version) "" + else s" ($version for ${dep.version})" - ( - Seq( - dep.module.organization, - dep.module.name, - dep.attributes.`type` - ) ++ - Some(dep.attributes.classifier) - .filter(_.nonEmpty) - .toSeq ++ + ( Seq( - version - ) - ).mkString(":") + extra - } - - if (res.conflicts.nonEmpty) { - // Needs test - println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") - } - - val errors = res.errors - - if (errors.nonEmpty) { - println(s"\n${errors.size} error(s):") - for ((dep, errs) <- errors) { - println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") + dep.module.organization, + dep.module.name, + dep.attributes.`type` + ) ++ + Some(dep.attributes.classifier) + .filter(_.nonEmpty) + .toSeq ++ + Seq( + version + ) + ).mkString(":") + extra } - throw new Exception(s"Encountered ${errors.length} error(s)") - } - val classifiers = - if (withClassifiers) - Some { - if (sbtClassifiers) - cm.classifiers - else - transitiveClassifiers.value + if (res.conflicts.nonEmpty) { + // Needs test + println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") + } + + val errors = res.errors + + if (errors.nonEmpty) { + println(s"\n${errors.size} error(s):") + for ((dep, errs) <- errors) { + println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") } - else - None - - val allArtifacts = - classifiers match { - case None => res.artifacts - case Some(cl) => res.classifiersArtifacts(cl) + throw new Exception(s"Encountered ${errors.length} error(s)") } - val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(logger), pool = pool).run.map((a, _)) - } - - if (verbosity >= 0) - errPrintln(s"Fetching artifacts") - - val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { - case -\/(ex) => - throw new Exception(s"Error while downloading / verifying artifacts", ex) - case \/-(l) => - l.toMap - } - - if (verbosity >= 0) - errPrintln(s"Fetching artifacts: done") - - val configs = { - val configs0 = ivyConfigurations.value.map { config => - config.name -> config.extendsConfigs.map(_.name) - }.toMap - - def allExtends(c: String) = { - // possibly bad complexity - def helper(current: Set[String]): Set[String] = { - val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) - if ((newSet -- current).nonEmpty) - helper(newSet) - else - newSet - } - - helper(Set(c)) - } - - configs0.map { - case (config, _) => - config -> allExtends(config) - } - } - - def artifactFileOpt(artifact: Artifact) = { - val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded")) - - fileOrError match { - case \/-(file) => - if (file.toString.contains("file:/")) - throw new Exception(s"Wrong path: $file") - Some(file) - case -\/(err) => - errPrintln(s"${artifact.url}: $err") + val classifiers = + if (withClassifiers) + Some { + if (sbtClassifiers) + cm.classifiers + else + transitiveClassifiers.value + } + else None + + val allArtifacts = + classifiers match { + case None => res.artifacts + case Some(cl) => res.classifiersArtifacts(cl) + } + + val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => + Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(logger), pool = pool).run.map((a, _)) } + + if (verbosity >= 0) + errPrintln(s"Fetching artifacts") + + val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { + case -\/(ex) => + throw new Exception(s"Error while downloading / verifying artifacts", ex) + case \/-(l) => + l.toMap + } + + if (verbosity >= 0) + errPrintln(s"Fetching artifacts: done") + + val configs = { + val configs0 = ivyConfigurations.value.map { config => + config.name -> config.extendsConfigs.map(_.name) + }.toMap + + def allExtends(c: String) = { + // possibly bad complexity + def helper(current: Set[String]): Set[String] = { + val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) + if ((newSet -- current).nonEmpty) + helper(newSet) + else + newSet + } + + helper(Set(c)) + } + + configs0.map { + case (config, _) => + config -> allExtends(config) + } + } + + def artifactFileOpt(artifact: Artifact) = { + val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded")) + + fileOrError match { + case \/-(file) => + if (file.toString.contains("file:/")) + throw new Exception(s"Wrong path: $file") + Some(file) + case -\/(err) => + errPrintln(s"${artifact.url}: $err") + None + } + } + + val depsByConfig = grouped(currentProject.dependencies) + + ToSbt.updateReport( + depsByConfig, + res, + configs, + classifiers, + artifactFileOpt + ) } - val depsByConfig = grouped(currentProject.dependencies) - - ToSbt.updateReport( - depsByConfig, - res, - configs, - classifiers, - artifactFileOpt + resolutionsCache.getOrElseUpdate( + CacheKey(startRes.copy(filter = None), withClassifiers, sbtClassifiers), + report ) } } From da98fbca2b45070d88f3097059775448cd4442d7 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:43 +0100 Subject: [PATCH 50/80] Make private methods that should be --- cache/src/main/scala/coursier/Cache.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index a9623d16d..af82d2da3 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -13,7 +13,7 @@ import java.io.{ Serializable => _, _ } object Cache { - def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = { + private def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = { def local(url: String) = if (url.startsWith("file:///")) url.stripPrefix("file://") @@ -48,7 +48,7 @@ object Cache { )) } - def download( + private def download( artifact: Artifact, cache: Seq[(String, File)], checksums: Set[String], From aa2a909fbd46e89eaed8b5d295b7b50c196cbb29 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:43 +0100 Subject: [PATCH 51/80] ResolutionClassLoader (WIP) --- .../main/scala/coursier/cli/Coursier.scala | 29 ++++++++++++++- .../coursier/ResolutionClassLoader.scala | 37 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 core/jvm/src/main/scala/coursier/ResolutionClassLoader.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 0f9a6e745..4bc3d55d5 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -97,6 +97,9 @@ case class Launch( @ExtraName("M") @ExtraName("main") mainClass: String, + @ExtraName("c") + @HelpMessage("Assume coursier is a dependency of the launched app, and share the coursier dependency of the launcher with it - allows the launched app to get the resolution that launched it via ResolutionClassLoader") + addCoursier: Boolean, @Recurse common: CommonOptions ) extends CoursierCommand { @@ -110,7 +113,29 @@ case class Launch( } } - val helper = new Helper(common, rawDependencies) + val extraForceVersions = + if (addCoursier) + ??? + else + Seq.empty[String] + + val dontFilterOut = + if (addCoursier) { + val url = classOf[coursier.core.Resolution].getProtectionDomain.getCodeSource.getLocation + + if (url.getProtocol == "file") + Seq(new File(url.getPath)) + else { + Console.err.println(s"Cannot get the location of the JAR of coursier ($url not a file URL)") + sys.exit(255) + } + } else + Seq.empty[File] + + val helper = new Helper( + common.copy(forceVersion = common.forceVersion ++ extraForceVersions), + rawDependencies + ) val files0 = helper.fetch(sources = false, javadoc = false) @@ -118,7 +143,7 @@ case class Launch( files0.map(_.toURI.toURL).toArray, new ClasspathFilter( Thread.currentThread().getContextClassLoader, - Coursier.baseCp.map(new File(_)).toSet, + Coursier.baseCp.map(new File(_)).toSet -- dontFilterOut, exclude = true ) ) diff --git a/core/jvm/src/main/scala/coursier/ResolutionClassLoader.scala b/core/jvm/src/main/scala/coursier/ResolutionClassLoader.scala new file mode 100644 index 000000000..3f852bb9c --- /dev/null +++ b/core/jvm/src/main/scala/coursier/ResolutionClassLoader.scala @@ -0,0 +1,37 @@ +package coursier + +import java.io.File +import java.net.URLClassLoader + +import coursier.util.ClasspathFilter + +class ResolutionClassLoader( + val resolution: Resolution, + val artifacts: Seq[(Dependency, Artifact, File)], + parent: ClassLoader +) extends URLClassLoader( + artifacts.map { case (_, _, f) => f.toURI.toURL }.toArray, + parent +) { + + /** + * Filtered version of this `ClassLoader`, exposing only `dependencies` and their + * their transitive dependencies, and filtering out the other dependencies from + * `resolution` - for `ClassLoader` isolation. + * + * An application launched by `coursier launch -C` has `ResolutionClassLoader` set as its + * context `ClassLoader` (can be obtain with `Thread.currentThread().getContextClassLoader`). + * If it aims at doing `ClassLoader` isolation, exposing only a dependency `dep` to the isolated + * things, `filter(dep)` provides a `ClassLoader` that loaded `dep` and all its transitive + * dependencies through the same loader as the contextual one, but that "exposes" only + * `dep` and its transitive dependencies, nothing more. + */ + def filter(dependencies: Set[Dependency]): ClassLoader = { + val subRes = resolution.subset(dependencies) + val subArtifacts = subRes.dependencyArtifacts.map { case (_, a) => a }.toSet + val subFiles = artifacts.collect { case (_, a, f) if subArtifacts(a) => f } + + new ClasspathFilter(this, subFiles.toSet, exclude = false) + } + +} From ebd97b8340ecd9fb06c582219b91a75cf2809d98 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:44 +0100 Subject: [PATCH 52/80] Move TermDisplay to cache module, remove dependency of plugin towards cli --- build.sbt | 6 +++--- .../cli => cache/src/main/scala/coursier}/TermDisplay.scala | 4 +--- plugin/src/main/scala/coursier/Tasks.scala | 1 - 3 files changed, 4 insertions(+), 7 deletions(-) rename {cli/src/main/scala/coursier/cli => cache/src/main/scala/coursier}/TermDisplay.scala (99%) diff --git a/build.sbt b/build.sbt index 99e0aafeb..cf03c9cd6 100644 --- a/build.sbt +++ b/build.sbt @@ -138,7 +138,8 @@ lazy val cache = project .settings( name := "coursier-cache", libraryDependencies ++= Seq( - "org.scalaz" %% "scalaz-concurrent" % "7.1.2" + "org.scalaz" %% "scalaz-concurrent" % "7.1.2", + "com.lihaoyi" %% "ammonite-terminal" % "0.5.0" ) ) @@ -168,7 +169,6 @@ lazy val cli = project name := "coursier-cli", libraryDependencies ++= Seq( "com.github.alexarchambault" %% "case-app" % "1.0.0-SNAPSHOT", - "com.lihaoyi" %% "ammonite-terminal" % "0.5.0", "ch.qos.logback" % "logback-classic" % "1.1.3" ) ) @@ -216,7 +216,7 @@ lazy val doc = project // Don't try to compile that if you're not in 2.10 lazy val plugin = project - .dependsOn(coreJvm, cache, cli) + .dependsOn(coreJvm, cache) .settings(baseCommonSettings) .settings( name := "coursier-sbt-plugin", diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala similarity index 99% rename from cli/src/main/scala/coursier/cli/TermDisplay.scala rename to cache/src/main/scala/coursier/TermDisplay.scala index 96f1d2695..f78cf7200 100644 --- a/cli/src/main/scala/coursier/cli/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -1,12 +1,10 @@ -package coursier.cli +package coursier import java.io.{File, Writer} import java.util.concurrent._ import ammonite.terminal.{ TTY, Ansi } -import coursier.Cache - import scala.annotation.tailrec import scala.collection.mutable.ArrayBuffer diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index a441a680f..1e6290565 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -3,7 +3,6 @@ package coursier import java.io.{ OutputStreamWriter, File } import java.util.concurrent.Executors -import coursier.cli.TermDisplay import coursier.ivy.IvyRepository import coursier.Keys._ import coursier.Structure._ From d33ab9da2661cbfcc46d45cc66e427819d29eead Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:44 +0100 Subject: [PATCH 53/80] Output better `sbt.ModuleReport`s --- .../scala/coursier/core/Definitions.scala | 24 +++- .../src/main/scala/coursier/ivy/IvyXml.scala | 30 ++++- .../src/main/scala/coursier/maven/Pom.scala | 98 +++++++++----- .../src/main/scala/coursier/package.scala | 3 + .../src/main/scala/coursier/util/Xml.scala | 15 +++ plugin/src/main/scala/coursier/FromSbt.scala | 3 +- plugin/src/main/scala/coursier/ToSbt.scala | 120 ++++++++++++++---- .../test/scala/coursier/test/package.scala | 3 +- 8 files changed, 233 insertions(+), 63 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 089e0f490..3d85be538 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -76,7 +76,10 @@ case class Project( // Ivy-specific // First String is configuration - publications: Seq[(String, Publication)] + publications: Seq[(String, Publication)], + + // Extra infos, not used during resolution + info: Info ) { def moduleVersion = (module, version) @@ -85,6 +88,25 @@ case class Project( Orders.allConfigurations(configurations) } +/** Extra project info, not used during resolution */ +case class Info( + description: String, + homePage: String, + licenses: Seq[(String, Option[String])], + developers: Seq[Info.Developer], + publication: Option[Versions.DateTime] +) + +object Info { + case class Developer( + id: String, + name: String, + url: String + ) + + val empty = Info("", "", Nil, Nil, None) +} + // Maven-specific case class Activation(properties: Seq[(String, Option[String])]) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala index 7725817bc..467c8dd48 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -106,7 +106,25 @@ object IvyXml { publicationsOpt = publicationsNodeOpt.map(publications) - } yield + } yield { + + val description = infoNode.children + .find(_.label == "description") + .map(_.textContent.trim) + .getOrElse("") + + val licenses = infoNode.children + .filter(_.label == "license") + .flatMap { n => + n.attribute("name").toOption.map { name => + (name, n.attribute("url").toOption) + }.toSeq + } + + val publicationDate = infoNode.attribute("publication") + .toOption + .flatMap(parseDateTime) + Project( module, version, @@ -128,7 +146,15 @@ object IvyXml { configurations0.flatMap { case (conf, _) => (publicationsOpt.flatMap(_.get(conf)).getOrElse(Nil) ++ inAllConfs).map(conf -> _) } - } + }, + Info( + description, + "", + licenses, + Nil, + publicationDate + ) ) + } } \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 06ece5f2f..f7d893a5d 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -177,39 +177,73 @@ object Pom { (mod.copy(attributes = Map.empty), ver) -> mod.attributes }.toMap - } yield Project( - projModule.copy(organization = groupId), - version, - deps.map { - case (config, dep0) => - val dep = extraAttrsMap.get(dep0.moduleVersion).fold(dep0)(attrs => - dep0.copy(module = dep0.module.copy(attributes = attrs)) - ) - config -> dep - }, - Map.empty, - parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), - depMgmts, - properties.toMap, - profiles, - None, - None, - Nil - ) - } + } yield { - def parseDateTime(s: String): Option[Versions.DateTime] = - if (s.length == 14 && s.forall(_.isDigit)) - Some(Versions.DateTime( - s.substring(0, 4).toInt, - s.substring(4, 6).toInt, - s.substring(6, 8).toInt, - s.substring(8, 10).toInt, - s.substring(10, 12).toInt, - s.substring(12, 14).toInt - )) - else - None + val description = pom.children + .find(_.label == "description") + .map(_.textContent) + .getOrElse("") + + val homePage = pom.children + .find(_.label == "url") + .map(_.textContent) + .getOrElse("") + + val licenses = pom.children + .find(_.label == "licenses") + .toSeq + .flatMap(_.children) + .filter(_.label == "license") + .flatMap { n => + n.attribute("name").toOption.map { name => + (name, n.attribute("url").toOption) + }.toSeq + } + + val developers = pom.children + .find(_.label == "developers") + .toSeq + .flatMap(_.children) + .filter(_.label == "developer") + .map { n => + for { + id <- n.attribute("id") + name <- n.attribute("name") + url <- n.attribute("url") + } yield Info.Developer(id, name, url) + } + .collect { + case \/-(d) => d + } + + Project( + projModule.copy(organization = groupId), + version, + deps.map { + case (config, dep0) => + val dep = extraAttrsMap.get(dep0.moduleVersion).fold(dep0)(attrs => + dep0.copy(module = dep0.module.copy(attributes = attrs)) + ) + config -> dep + }, + Map.empty, + parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), + depMgmts, + properties.toMap, + profiles, + None, + None, + Nil, + Info( + description, + homePage, + licenses, + developers, + None + ) + ) + } + } def versions(node: Node): String \/ Versions = { import Scalaz._ diff --git a/core/shared/src/main/scala/coursier/package.scala b/core/shared/src/main/scala/coursier/package.scala index ec15aabc6..8a19dd3cc 100644 --- a/core/shared/src/main/scala/coursier/package.scala +++ b/core/shared/src/main/scala/coursier/package.scala @@ -37,6 +37,9 @@ package object coursier { type Project = core.Project val Project = core.Project + type Info = core.Info + val Info = core.Info + type Profile = core.Profile val Profile = core.Profile diff --git a/core/shared/src/main/scala/coursier/util/Xml.scala b/core/shared/src/main/scala/coursier/util/Xml.scala index 83fe20f9b..a7469d659 100644 --- a/core/shared/src/main/scala/coursier/util/Xml.scala +++ b/core/shared/src/main/scala/coursier/util/Xml.scala @@ -1,5 +1,7 @@ package coursier.util +import coursier.core.Versions + import scalaz.{\/-, -\/, \/, Scalaz} object Xml { @@ -55,4 +57,17 @@ object Xml { .toRightDisjunction(s"$description not found") } + def parseDateTime(s: String): Option[Versions.DateTime] = + if (s.length == 14 && s.forall(_.isDigit)) + Some(Versions.DateTime( + s.substring(0, 4).toInt, + s.substring(4, 6).toInt, + s.substring(6, 8).toInt, + s.substring(8, 10).toInt, + s.substring(10, 12).toInt, + s.substring(12, 14).toInt + )) + else + None + } diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index b2e22ba00..ee4940471 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -101,7 +101,8 @@ object FromSbt { Nil, None, None, - Nil + Nil, + Info.empty ) } diff --git a/plugin/src/main/scala/coursier/ToSbt.scala b/plugin/src/main/scala/coursier/ToSbt.scala index 21b8308c3..8839f8ffa 100644 --- a/plugin/src/main/scala/coursier/ToSbt.scala +++ b/plugin/src/main/scala/coursier/ToSbt.scala @@ -1,5 +1,7 @@ package coursier +import java.util.GregorianCalendar + import sbt._ object ToSbt { @@ -24,33 +26,61 @@ object ToSbt { Map.empty ) - def moduleReport(dependency: Dependency, artifacts: Seq[(Artifact, Option[File])]): sbt.ModuleReport = + def moduleReport( + dependency: Dependency, + dependees: Seq[(Dependency, Project)], + project: Project, + artifacts: Seq[(Artifact, Option[File])] + ): sbt.ModuleReport = { + + val sbtArtifacts = artifacts.collect { + case (artifact, Some(file)) => + (ToSbt.artifact(dependency.module, artifact), file) + } + val sbtMissingArtifacts = artifacts.collect { + case (artifact, None) => + ToSbt.artifact(dependency.module, artifact) + } + + val publicationDate = project.info.publication.map { dt => + new GregorianCalendar(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second).getTime + } + + val callers = dependees.map { + case (dependee, dependeeProj) => + new Caller( + ToSbt.moduleId(dependee), + dependeeProj.configurations.keys.toVector, + dependee.module.attributes ++ dependeeProj.properties, + // FIXME Set better values here + isForceDependency = false, + isChangingDependency = false, + isTransitiveDependency = false, + isDirectlyForceDependency = false + ) + } + new sbt.ModuleReport( - ToSbt.moduleId(dependency), - artifacts.collect { - case (artifact, Some(file)) => - (ToSbt.artifact(dependency.module, artifact), file) - }, - artifacts.collect { - case (artifact, None) => - ToSbt.artifact(dependency.module, artifact) - }, - None, - None, - None, - None, - false, - None, - None, - None, - None, - Map.empty, - None, - None, - Nil, - Nil, - Nil + module = ToSbt.moduleId(dependency), + artifacts = sbtArtifacts, + missingArtifacts = sbtMissingArtifacts, + status = None, + publicationDate = publicationDate, + resolver = None, + artifactResolver = None, + evicted = false, + evictedData = None, + evictedReason = None, + problem = None, + homepage = Some(project.info.homePage).filter(_.nonEmpty), + extraAttributes = dependency.module.attributes ++ project.properties, + isDefault = None, + branch = None, + configurations = project.configurations.keys.toVector, + licenses = project.info.licenses, + callers = callers ) + } private def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = map.groupBy { case (k, _) => k }.map { @@ -71,9 +101,47 @@ object ToSbt { val groupedDepArtifacts = grouped(depArtifacts) + val versions = res.dependencies.toVector.map { dep => + dep.module -> dep.version + }.toMap + + def clean(dep: Dependency): Dependency = + dep.copy(configuration = "", exclusions = Set.empty, optional = false) + + val reverseDependencies = res.reverseDependencies + .toVector + .map { case (k, v) => + clean(k) -> v.map(clean) + } + .groupBy { case (k, v) => k } + .mapValues { v => + v.flatMap { + case (_, l) => l + } + } + .toVector + .toMap + groupedDepArtifacts.map { case (dep, artifacts) => - ToSbt.moduleReport(dep, artifacts.map(a => a -> artifactFileOpt(a))) + val (_, proj) = res.projectCache(dep.moduleVersion) + + // FIXME Likely flaky... + val dependees = reverseDependencies + .getOrElse(clean(dep.copy(version = "")), Vector.empty) + .map { dependee0 => + val version = versions(dependee0.module) + val dependee = dependee0.copy(version = version) + val (_, dependeeProj) = res.projectCache(dependee.moduleVersion) + (dependee, dependeeProj) + } + + ToSbt.moduleReport( + dep, + dependees, + proj, + artifacts.map(a => a -> artifactFileOpt(a)) + ) } } diff --git a/tests/shared/src/test/scala/coursier/test/package.scala b/tests/shared/src/test/scala/coursier/test/package.scala index 83d17cf7d..1bb6d4487 100644 --- a/tests/shared/src/test/scala/coursier/test/package.scala +++ b/tests/shared/src/test/scala/coursier/test/package.scala @@ -56,7 +56,8 @@ package object test { profiles, versions, snapshotVersioning, - publications + publications, + Info.empty ) } } From 138d565187a8a5e320f297eb338d5320f57d5be9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:44 +0100 Subject: [PATCH 54/80] Write Ivy related files in resolution cache for publish to be fine --- .../main/scala/coursier/CoursierPlugin.scala | 2 + plugin/src/main/scala/coursier/FromSbt.scala | 16 +++- plugin/src/main/scala/coursier/Keys.scala | 2 + .../src/main/scala/coursier/MakeIvyXml.scala | 64 ++++++++++++++ plugin/src/main/scala/coursier/Tasks.scala | 87 ++++++++++++++++++- 5 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 plugin/src/main/scala/coursier/MakeIvyXml.scala diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala index cd9e4ebc3..7d617f7f4 100644 --- a/plugin/src/main/scala/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -23,6 +23,7 @@ object CoursierPlugin extends AutoPlugin { val coursierCache = Keys.coursierCache val coursierProject = Keys.coursierProject val coursierProjects = Keys.coursierProjects + val coursierPublications = Keys.coursierPublications val coursierSbtClassifiersModule = Keys.coursierSbtClassifiersModule } @@ -44,6 +45,7 @@ object CoursierPlugin extends AutoPlugin { updateSbtClassifiers in Defaults.TaskGlobal <<= Tasks.updateTask(withClassifiers = true, sbtClassifiers = true), coursierProject <<= Tasks.coursierProjectTask, coursierProjects <<= Tasks.coursierProjectsTask, + coursierPublications <<= Tasks.coursierPublicationsTask, coursierSbtClassifiersModule <<= classifiersModule in updateSbtClassifiers ) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index ee4940471..0611e2ed2 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -10,10 +10,18 @@ object FromSbt { moduleId: ModuleID, scalaVersion: => String, scalaBinaryVersion: => String - ): String = moduleId.crossVersion match { - case CrossVersion.Disabled => moduleId.name - case f: CrossVersion.Full => moduleId.name + "_" + f.remapVersion(scalaVersion) - case f: CrossVersion.Binary => moduleId.name + "_" + f.remapVersion(scalaBinaryVersion) + ): String = + sbtCrossVersionName(moduleId.name, moduleId.crossVersion, scalaVersion, scalaBinaryVersion) + + def sbtCrossVersionName( + name: String, + crossVersion: CrossVersion, + scalaVersion: => String, + scalaBinaryVersion: => String + ): String = crossVersion match { + case CrossVersion.Disabled => name + case f: CrossVersion.Full => name + "_" + f.remapVersion(scalaVersion) + case f: CrossVersion.Binary => name + "_" + f.remapVersion(scalaBinaryVersion) } def mappings(mapping: String): Seq[(String, String)] = diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala index 6fa28d804..3ae648fe3 100644 --- a/plugin/src/main/scala/coursier/Keys.scala +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -1,6 +1,7 @@ package coursier import java.io.File +import coursier.core.Publication import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey } object Keys { @@ -19,6 +20,7 @@ object Keys { val coursierProject = TaskKey[(Project, Seq[(String, Seq[Artifact])])]("coursier-project", "") val coursierProjects = TaskKey[Seq[(Project, Seq[(String, Seq[Artifact])])]]("coursier-projects", "") + val coursierPublications = TaskKey[Seq[(String, Publication)]]("coursier-publications", "") val coursierSbtClassifiersModule = TaskKey[GetClassifiersModule]("coursier-sbt-classifiers-module", "") } diff --git a/plugin/src/main/scala/coursier/MakeIvyXml.scala b/plugin/src/main/scala/coursier/MakeIvyXml.scala new file mode 100644 index 000000000..405e69af4 --- /dev/null +++ b/plugin/src/main/scala/coursier/MakeIvyXml.scala @@ -0,0 +1,64 @@ +package coursier + +import scala.xml.{ Node, PrefixedAttribute } + +object MakeIvyXml { + + def apply(project: Project): Node = { + + val baseInfoAttrs = .attributes + + val infoAttrs = project.module.attributes.foldLeft(baseInfoAttrs) { + case (acc, (k, v)) => + new PrefixedAttribute("e", k, v, acc) + } + + val licenseElems = project.info.licenses.map { + case (name, urlOpt) => + var n = + for (url <- urlOpt) + n = n % .attributes + n + } + + val infoElem = { + + {licenseElems} + {project.info.description} + + } % infoAttrs + + val confElems = project.configurations.toVector.map { + case (name, extends0) => + var n = + if (extends0.nonEmpty) + n = n % .attributes + n + } + + val publicationElems = project.publications.map { + case (conf, pub) => + var n = + if (pub.classifier.nonEmpty) + n = n % .attributes + n + } + + val dependencyElems = project.dependencies.toVector.map { + case (conf, dep) => + ${dep.configuration}"} /> + } + + + {infoElem} + {confElems} + {publicationElems} + {dependencyElems} + + } + +} diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 1e6290565..6a079ba00 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -1,16 +1,21 @@ package coursier import java.io.{ OutputStreamWriter, File } +import java.nio.file.Files import java.util.concurrent.Executors +import coursier.core.Publication import coursier.ivy.IvyRepository import coursier.Keys._ import coursier.Structure._ +import org.apache.ivy.core.module.id.ModuleRevisionId import sbt.{ UpdateReport, Classpaths, Resolver, Def } +import sbt.Configurations.{ Compile, Test } import sbt.Keys._ import scala.collection.mutable +import scala.collection.JavaConverters._ import scalaz.{ \/-, -\/ } import scalaz.concurrent.{ Task, Strategy } @@ -76,6 +81,51 @@ object Tasks { coursierProject.forAllProjects(state, projects).map(_.values.toVector) } + def coursierPublicationsTask: Def.Initialize[sbt.Task[Seq[(String, Publication)]]] = + ( + sbt.Keys.state, + sbt.Keys.thisProjectRef, + sbt.Keys.projectID, + sbt.Keys.scalaVersion, + sbt.Keys.scalaBinaryVersion + ).map { (state, projectRef, projId, sv, sbv) => + + val packageTasks = Seq(packageBin, packageSrc, packageDoc) + val configs = Seq(Compile, Test) + + val sbtArtifacts = + for { + pkgTask <- packageTasks + config <- configs + } yield { + val publish = publishArtifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, false) + if (publish) + Option(artifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, null)) + .map(config.name -> _) + else + None + } + + sbtArtifacts.collect { + case Some((config, artifact)) => + val name = FromSbt.sbtCrossVersionName( + artifact.name, + projId.crossVersion, + sv, + sbv + ) + + val publication = Publication( + name, + artifact.`type`, + artifact.extension, + artifact.classifier.getOrElse("") + ) + + config -> publication + } + } + // FIXME More things should possibly be put here too (resolvers, etc.) private case class CacheKey( resolution: Resolution, @@ -112,10 +162,25 @@ object Tasks { scalaBinaryVersion.value ) else { - val (p, _) = coursierProject.value - p + val (proj, _) = coursierProject.value + val publications = coursierPublications.value + proj.copy(publications = publications) } + val ivySbt0 = ivySbt.value + val ivyCacheManager = ivySbt0.withIvy(streams.value.log)(ivy => + ivy.getResolutionCacheManager + ) + + val ivyModule = ModuleRevisionId.newInstance( + currentProject.module.organization, + currentProject.module.name, + currentProject.version, + currentProject.module.attributes.asJava + ) + val cacheIvyFile = ivyCacheManager.getResolvedIvyFileInCache(ivyModule) + val cacheIvyPropertiesFile = ivyCacheManager.getResolvedIvyPropertiesInCache(ivyModule) + val projects = coursierProjects.value val parallelDownloads = coursierParallelDownloads.value @@ -140,6 +205,22 @@ object Tasks { forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap ) + // required for publish to be fine, later on + def writeIvyFiles() = { + val printer = new scala.xml.PrettyPrinter(80, 2) + + val b = new StringBuilder + b ++= """""" + b += '\n' + b ++= printer.format(MakeIvyXml(currentProject)) + cacheIvyFile.getParentFile.mkdirs() + Files.write(cacheIvyFile.toPath, b.result().getBytes("UTF-8")) + + // Just writing an empty file here... Are these only used? + cacheIvyPropertiesFile.getParentFile.mkdirs() + Files.write(cacheIvyPropertiesFile.toPath, "".getBytes("UTF-8")) + } + def report = { if (verbosity >= 1) { println("InterProjectRepository") @@ -320,6 +401,8 @@ object Tasks { val depsByConfig = grouped(currentProject.dependencies) + writeIvyFiles() + ToSbt.updateReport( depsByConfig, res, From d066fd2c29474d0e3901dd7f8e332e186489cf25 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:44 +0100 Subject: [PATCH 55/80] Add back bootstrap JAR in CLI package --- build.sbt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index cf03c9cd6..26e97d72a 100644 --- a/build.sbt +++ b/build.sbt @@ -170,7 +170,10 @@ lazy val cli = project libraryDependencies ++= Seq( "com.github.alexarchambault" %% "case-app" % "1.0.0-SNAPSHOT", "ch.qos.logback" % "logback-classic" % "1.1.3" - ) + ), + resourceGenerators in Compile += packageBin.in(bootstrap).in(Compile).map { jar => + Seq(jar) + }.taskValue ) lazy val web = project From 6a460c1fb245ebb1c29d121336c54692932b8e48 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:45 +0100 Subject: [PATCH 56/80] Fix in ClasspathFilter --- .../scala/coursier/util/ClasspathFilter.scala | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala b/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala index 4f7cafe4a..7272a2723 100644 --- a/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala +++ b/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala @@ -84,9 +84,23 @@ class ClasspathFilter(parent: ClassLoader, classpath: Set[File], exclude: Boolea override def loadClass(className: String, resolve: Boolean): Class[_] = { - val c = super.loadClass(className, resolve) - if (fromClasspath(c)) c - else throw new ClassNotFoundException(className) + val c = + try super.loadClass(className, resolve) + catch { + case e: LinkageError => + // Happens when trying to derive a shapeless.Generic + // from an Ammonite session launched like + // ./coursier launch com.lihaoyi:ammonite-repl_2.11.7:0.5.2 + // For className == "shapeless.GenericMacros", + // the super.loadClass above - which would be filtered out below anyway, + // raises a NoClassDefFoundError. + null + } + + if (c != null && fromClasspath(c)) + c + else + throw new ClassNotFoundException(className) } override def getResource(name: String): URL = { From f4c82ece40080ae78f98c1d0eb15c47b6a3b4600 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:45 +0100 Subject: [PATCH 57/80] Switch to case-app 1.0.0-M1-SNAPSHOT, disable cli build in 2.10 --- build.sbt | 4 ++-- project/travis.sh | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 26e97d72a..ff9e10dc6 100644 --- a/build.sbt +++ b/build.sbt @@ -64,7 +64,6 @@ lazy val baseCommonSettings = Seq( lazy val commonSettings = baseCommonSettings ++ Seq( scalaVersion := "2.11.7", - crossScalaVersions := Seq("2.10.6", "2.11.7"), libraryDependencies ++= { if (scalaVersion.value startsWith "2.10.") Seq(compilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full)) @@ -168,7 +167,8 @@ lazy val cli = project .settings( name := "coursier-cli", libraryDependencies ++= Seq( - "com.github.alexarchambault" %% "case-app" % "1.0.0-SNAPSHOT", + // beware - available only in 2.11 + "com.github.alexarchambault" %% "case-app" % "1.0.0-M1-SNAPSHOT", "ch.qos.logback" % "logback-classic" % "1.1.3" ), resourceGenerators in Compile += packageBin.in(bootstrap).in(Compile).map { jar => diff --git a/project/travis.sh b/project/travis.sh index 25fddad7d..32116325c 100755 --- a/project/travis.sh +++ b/project/travis.sh @@ -29,8 +29,10 @@ function isMasterOrDevelop() { # web sub-project doesn't compile in 2.10 (no scalajs-react) if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then + IS_210=1 SBT_COMMANDS="cli/compile" else + IS_210=0 SBT_COMMANDS="compile" fi @@ -43,7 +45,12 @@ SBT_COMMANDS="$SBT_COMMANDS test" PUSH_GHPAGES=0 if isNotPr && publish && isMaster; then - SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish files/publish cli/publish" + SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish files/publish" + if [ "$IS_210" = 1 ]; then + SBT_COMMANDS="$SBT_COMMANDS plugin/publish" + else + SBT_COMMANDS="$SBT_COMMANDS cli/publish" + fi fi if isNotPr && publish && isMasterOrDevelop; then From 9c9c6de14e127c2ac97d37496880001a65123c41 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:45 +0100 Subject: [PATCH 58/80] Remove debug message --- cli/src/main/scala/coursier/cli/Helper.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 1726d8b7c..48b01a276 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -180,7 +180,6 @@ class Helper( val attributes = splitAttributes.collect { case Seq(k, v) => k -> v } - println(s"-> $org:$name:$attributes") (Module(org, name, attributes.toMap), version) } From 1a51200ec59c309b67be8a0287ba53411e084b9f Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:45 +0100 Subject: [PATCH 59/80] More verbose plugin output --- plugin/src/main/scala/coursier/Tasks.scala | 30 +++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 6a079ba00..4ac4eed3f 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -263,14 +263,31 @@ object Tasks { Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(logger), pool = pool) ) - def depsRepr = currentProject.dependencies.map { case (config, dep) => - s"${dep.module}:${dep.version}:$config->${dep.configuration}" - }.sorted + def depsRepr(deps: Seq[(String, Dependency)]) = + deps.map { case (config, dep) => + s"${dep.module}:${dep.version}:$config->${dep.configuration}" + }.sorted.distinct + + def depsRepr0(deps: Seq[Dependency]) = + deps.map { dep => + s"${dep.module}:${dep.version}:${dep.configuration}" + }.sorted.distinct + + if (verbosity >= 1) { + errPrintln(s"Repositories:") + val repositories0 = repositories.map { + case r: IvyRepository => r.copy(properties = Map.empty) + case r: InterProjectRepository => r.copy(projects = Nil) + case r => r + } + for (repo <- repositories0) + errPrintln(s" $repo") + } if (verbosity >= 0) errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") if (verbosity >= 1) - for (depRepr <- depsRepr.distinct) + for (depRepr <- depsRepr(currentProject.dependencies)) errPrintln(s" $depRepr") val res = startRes @@ -280,11 +297,16 @@ object Tasks { .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) .merge + + if (!res.isDone) throw new Exception(s"Maximum number of iteration reached!") if (verbosity >= 0) errPrintln("Resolution done") + if (verbosity >= 1) + for (depRepr <- depsRepr0(res.dependencies.toSeq)) + errPrintln(s" $depRepr") def repr(dep: Dependency) = { // dep.version can be an interval, whereas the one from project can't From 0d5ac09aa10890a4cf9a708b5cc75f0484fdcad6 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:46 +0100 Subject: [PATCH 60/80] More for extra attributes from POM --- .../main/scala/coursier/cli/Coursier.scala | 3 ++- cli/src/main/scala/coursier/cli/Helper.scala | 4 ++-- .../src/main/scala/coursier/Fetch.scala | 4 ++-- .../coursier/maven/MavenRepository.scala | 22 +++++++++++++------ .../scala/coursier/maven/MavenSource.scala | 2 +- plugin/src/main/scala/coursier/FromSbt.scala | 2 +- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 4bc3d55d5..40dc8ac42 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -31,7 +31,8 @@ case class CommonOptions( @HelpMessage("Do not add default repositories (~/.ivy2/local, and Central)") noDefault: Boolean = false, @HelpMessage("Modify names in Maven repository paths for SBT plugins") - sbtPluginHack: Boolean = false, + @ValueDescription("Attribute prefix (typically \"e\")") + sbtPluginHack: String = "", @HelpMessage("Force module version") @ValueDescription("organization:name:forcedVersion") @ExtraName("V") diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 48b01a276..4800b8f03 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -118,9 +118,9 @@ class Helper( repositories0.collect { case Right(r) => r } val repositories = - if (common.sbtPluginHack) + if (common.sbtPluginHack.nonEmpty) repositories1.map { - case m: MavenRepository => m.copy(sbtAttrStub = true) + case m: MavenRepository => m.copy(sbtAttrStub = Some(common.sbtPluginHack)) case other => other } else diff --git a/core/shared/src/main/scala/coursier/Fetch.scala b/core/shared/src/main/scala/coursier/Fetch.scala index 65d393ad3..ffaef450e 100644 --- a/core/shared/src/main/scala/coursier/Fetch.scala +++ b/core/shared/src/main/scala/coursier/Fetch.scala @@ -42,7 +42,7 @@ object Fetch { val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { case (acc, (repo, eitherProjTask)) => val looseModuleValidation = repo match { - case m: MavenRepository => m.sbtAttrStub // that sucks so much + case m: MavenRepository => m.sbtAttrStub.nonEmpty // that sucks so much case _ => false } val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module @@ -66,7 +66,7 @@ object Fetch { EitherT(F.map(task)(_.leftMap(_.reverse))) .map {case x @ (source, proj) => val looseModuleValidation = source match { - case m: MavenSource => m.sbtAttrStub // omfg + case m: MavenSource => m.sbtAttrStub.nonEmpty // omfg case _ => false } val projModule = diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 821cd1866..7092b073d 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -45,16 +45,24 @@ object MavenRepository { "test" -> Seq("runtime") ) - def dirModuleName(module: Module, sbtAttrStub: Boolean): String = - if (sbtAttrStub) { + def dirModuleName(module: Module, sbtAttrStub: Option[String]): String = + sbtAttrStub.fold(module.name) { prefix => + def attr(name: String) = { + val base = module.attributes.get(name) + + if (prefix.isEmpty) + base + else + base.orElse(module.attributes.get(s"$prefix:$name")) + } + var name = module.name - for (scalaVersion <- module.attributes.get("scalaVersion")) + for (scalaVersion <- attr("scalaVersion")) name = name + "_" + scalaVersion - for (sbtVersion <- module.attributes.get("sbtVersion")) + for (sbtVersion <- attr("sbtVersion")) name = name + "_" + sbtVersion name - } else - module.name + } } @@ -63,7 +71,7 @@ case class MavenRepository( ivyLike: Boolean = false, changing: Option[Boolean] = None, /** Hackish hack for sbt plugins mainly - what this does really sucks */ - sbtAttrStub: Boolean = false + sbtAttrStub: Option[String] = None ) extends Repository { import Repository._ diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index f07a4f5b3..193c7f4ba 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -7,7 +7,7 @@ case class MavenSource( ivyLike: Boolean, changing: Option[Boolean] = None, /** See doc on MavenRepository */ - sbtAttrStub: Boolean + sbtAttrStub: Option[String] = None ) extends Artifact.Source { import Repository._ diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 0611e2ed2..253a48c34 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -119,7 +119,7 @@ object FromSbt { case sbt.MavenRepository(_, root) => if (root.startsWith("http://") || root.startsWith("https://")) { val root0 = if (root.endsWith("/")) root else root + "/" - Some(MavenRepository(root0, sbtAttrStub = true)) + Some(MavenRepository(root0, sbtAttrStub = Some("e"))) } else { Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it") None From aa9a43b483c6ad1af3434414e9c1702e671ed785 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:46 +0100 Subject: [PATCH 61/80] More for extra attributes from POM (#2) --- .../main/scala/coursier/cli/Coursier.scala | 3 +-- cli/src/main/scala/coursier/cli/Helper.scala | 4 ++-- .../src/main/scala/coursier/Fetch.scala | 4 ++-- .../coursier/maven/MavenRepository.scala | 22 ++++++------------- .../scala/coursier/maven/MavenSource.scala | 2 +- .../src/main/scala/coursier/maven/Pom.scala | 4 +++- plugin/src/main/scala/coursier/FromSbt.scala | 2 +- 7 files changed, 17 insertions(+), 24 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 40dc8ac42..4bc3d55d5 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -31,8 +31,7 @@ case class CommonOptions( @HelpMessage("Do not add default repositories (~/.ivy2/local, and Central)") noDefault: Boolean = false, @HelpMessage("Modify names in Maven repository paths for SBT plugins") - @ValueDescription("Attribute prefix (typically \"e\")") - sbtPluginHack: String = "", + sbtPluginHack: Boolean = false, @HelpMessage("Force module version") @ValueDescription("organization:name:forcedVersion") @ExtraName("V") diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 4800b8f03..48b01a276 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -118,9 +118,9 @@ class Helper( repositories0.collect { case Right(r) => r } val repositories = - if (common.sbtPluginHack.nonEmpty) + if (common.sbtPluginHack) repositories1.map { - case m: MavenRepository => m.copy(sbtAttrStub = Some(common.sbtPluginHack)) + case m: MavenRepository => m.copy(sbtAttrStub = true) case other => other } else diff --git a/core/shared/src/main/scala/coursier/Fetch.scala b/core/shared/src/main/scala/coursier/Fetch.scala index ffaef450e..65d393ad3 100644 --- a/core/shared/src/main/scala/coursier/Fetch.scala +++ b/core/shared/src/main/scala/coursier/Fetch.scala @@ -42,7 +42,7 @@ object Fetch { val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { case (acc, (repo, eitherProjTask)) => val looseModuleValidation = repo match { - case m: MavenRepository => m.sbtAttrStub.nonEmpty // that sucks so much + case m: MavenRepository => m.sbtAttrStub // that sucks so much case _ => false } val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module @@ -66,7 +66,7 @@ object Fetch { EitherT(F.map(task)(_.leftMap(_.reverse))) .map {case x @ (source, proj) => val looseModuleValidation = source match { - case m: MavenSource => m.sbtAttrStub.nonEmpty // omfg + case m: MavenSource => m.sbtAttrStub // omfg case _ => false } val projModule = diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 7092b073d..821cd1866 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -45,24 +45,16 @@ object MavenRepository { "test" -> Seq("runtime") ) - def dirModuleName(module: Module, sbtAttrStub: Option[String]): String = - sbtAttrStub.fold(module.name) { prefix => - def attr(name: String) = { - val base = module.attributes.get(name) - - if (prefix.isEmpty) - base - else - base.orElse(module.attributes.get(s"$prefix:$name")) - } - + def dirModuleName(module: Module, sbtAttrStub: Boolean): String = + if (sbtAttrStub) { var name = module.name - for (scalaVersion <- attr("scalaVersion")) + for (scalaVersion <- module.attributes.get("scalaVersion")) name = name + "_" + scalaVersion - for (sbtVersion <- attr("sbtVersion")) + for (sbtVersion <- module.attributes.get("sbtVersion")) name = name + "_" + sbtVersion name - } + } else + module.name } @@ -71,7 +63,7 @@ case class MavenRepository( ivyLike: Boolean = false, changing: Option[Boolean] = None, /** Hackish hack for sbt plugins mainly - what this does really sucks */ - sbtAttrStub: Option[String] = None + sbtAttrStub: Boolean = false ) extends Repository { import Repository._ diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 193c7f4ba..f07a4f5b3 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -7,7 +7,7 @@ case class MavenSource( ivyLike: Boolean, changing: Option[Boolean] = None, /** See doc on MavenRepository */ - sbtAttrStub: Option[String] = None + sbtAttrStub: Boolean ) extends Artifact.Source { import Repository._ diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index f7d893a5d..433d90c84 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -382,6 +382,8 @@ object Pom { "branch" ) + val extraAttributeDropPrefix = "e:" + def extraAttribute(s: String): String \/ (Module, String) = { // vaguely does the same as: // https://github.com/apache/ant-ivy/blob/2.2.0/src/java/org/apache/ivy/core/module/id/ModuleRevisionId.java#L291 @@ -410,7 +412,7 @@ object Pom { parts <- partsOrError attrs = parts.grouped(2).collect { case Seq(k, v) if v != "NULL" => - k -> v + k.stripPrefix(extraAttributeDropPrefix) -> v }.toMap org <- attrFrom(attrs, extraAttributeOrg) name <- attrFrom(attrs, extraAttributeName) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 253a48c34..0611e2ed2 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -119,7 +119,7 @@ object FromSbt { case sbt.MavenRepository(_, root) => if (root.startsWith("http://") || root.startsWith("https://")) { val root0 = if (root.endsWith("/")) root else root + "/" - Some(MavenRepository(root0, sbtAttrStub = Some("e"))) + Some(MavenRepository(root0, sbtAttrStub = true)) } else { Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it") None From 00d11017f7d89ccc6ea55b16ae9c8b9b83f9516b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:46 +0100 Subject: [PATCH 62/80] Better properties substitution For com.fasterxml.jackson.jaxrs:jackson-jaxrs-providers:2.6.0 in particular, having properties refering other properties --- .../scala/coursier/core/Definitions.scala | 2 +- .../main/scala/coursier/core/Resolution.scala | 99 ++++++++++--------- .../src/main/scala/coursier/ivy/IvyXml.scala | 2 +- .../src/main/scala/coursier/maven/Pom.scala | 2 +- 4 files changed, 56 insertions(+), 49 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 3d85be538..de4604840 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -69,7 +69,7 @@ case class Project( // Maven-specific parent: Option[(Module, String)], dependencyManagement: Seq[(String, Dependency)], - properties: Map[String, String], + properties: Seq[(String, String)], profiles: Seq[Profile], versions: Option[Versions], snapshotVersioning: Option[SnapshotVersioning], diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index ea035dd40..5db05a72d 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -56,12 +56,6 @@ object Resolution { (dict /: deps)(add) } - def mergeProperties( - dict: Map[String, String], - other: Map[String, String] - ): Map[String, String] = - dict ++ other.filterKeys(!dict.contains(_)) - def addDependencies(deps: Seq[Seq[(String, Dependency)]]): Seq[(String, Dependency)] = { val res = (deps :\ (Set.empty[DepMgmt.Key], Seq.empty[(String, Dependency)])) { @@ -79,6 +73,32 @@ object Resolution { quote("${") + "([a-zA-Z0-9-.]*)" + quote("}") ).r + def substituteProps(s: String, properties: Map[String, String]) = { + val matches = propRegex + .findAllMatchIn(s) + .toList + .reverse + + if (matches.isEmpty) s + else { + val output = + (new StringBuilder(s) /: matches) { (b, m) => + properties + .get(m.group(1)) + .fold(b)(b.replace(m.start, m.end, _)) + } + + output.result() + } + } + + def propertiesMap(props: Seq[(String, String)]): Map[String, String] = + props.foldLeft(Map.empty[String, String]) { + case (acc, (k, v0)) => + val v = substituteProps(v0, acc) + acc + (k -> v) + } + /** * Substitutes `properties` in `dependencies`. */ @@ -87,41 +107,25 @@ object Resolution { properties: Map[String, String] ): Seq[(String, Dependency)] = { - def substituteProps(s: String) = { - val matches = propRegex - .findAllMatchIn(s) - .toList - .reverse - - if (matches.isEmpty) s - else { - val output = - (new StringBuilder(s) /: matches) { (b, m) => - properties - .get(m.group(1)) - .fold(b)(b.replace(m.start, m.end, _)) - } - - output.result() - } - } + def substituteProps0(s: String) = + substituteProps(s, properties) dependencies .map {case (config, dep) => - substituteProps(config) -> dep.copy( + substituteProps0(config) -> dep.copy( module = dep.module.copy( - organization = substituteProps(dep.module.organization), - name = substituteProps(dep.module.name) + organization = substituteProps0(dep.module.organization), + name = substituteProps0(dep.module.name) ), - version = substituteProps(dep.version), + version = substituteProps0(dep.version), attributes = dep.attributes.copy( - `type` = substituteProps(dep.attributes.`type`), - classifier = substituteProps(dep.attributes.classifier) + `type` = substituteProps0(dep.attributes.`type`), + classifier = substituteProps0(dep.attributes.classifier) ), - configuration = substituteProps(dep.configuration), + configuration = substituteProps0(dep.configuration), exclusions = dep.exclusions .map{case (org, name) => - (substituteProps(org), substituteProps(name)) + (substituteProps0(org), substituteProps0(name)) } // FIXME The content of the optional tag may also be a property in // the original POM. Maybe not parse it that earlier? @@ -320,15 +324,14 @@ object Resolution { // come from parents or dependency management. This may not be // the right thing to do. - val properties = mergeProperties( - project.properties, - Map( - "project.groupId" -> project.module.organization, - "project.artifactId" -> project.module.name, - "project.version" -> project.version - ) + val properties0 = project.properties ++ Seq( + "project.groupId" -> project.module.organization, + "project.artifactId" -> project.module.name, + "project.version" -> project.version ) + val properties = propertiesMap(properties0) + val configurations = withParentConfigurations(from.configuration, project.configurations) withExclusions( @@ -604,11 +607,13 @@ case class Resolution( project: Project ): Set[ModuleVersion] = { - val approxProperties = + val approxProperties0 = project.parent .flatMap(projectCache.get) .map(_._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) + .fold(project.properties)(project.properties ++ _) + + val approxProperties = propertiesMap(approxProperties0) val profileDependencies = profiles( @@ -678,11 +683,13 @@ case class Resolution( */ def withDependencyManagement(project: Project): Project = { - val approxProperties = + val approxProperties0 = project.parent .filter(projectCache.contains) - .map(projectCache(_)._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) + .map(projectCache(_)._2.properties.toMap) + .fold(project.properties)(project.properties ++ _) + + val approxProperties = propertiesMap(approxProperties0) val profiles0 = profiles( project, @@ -695,7 +702,7 @@ case class Resolution( ) val properties0 = (project.properties /: profiles0) { (acc, p) => - mergeProperties(acc, p.properties) + acc ++ p.properties } val deps = ( @@ -732,7 +739,7 @@ case class Resolution( properties = project.parent .filter(projectCache.contains) .map(projectCache(_)._2.properties) - .fold(properties0)(mergeProperties(properties0, _)) + .fold(properties0)(properties0 ++ _) ) } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala index 467c8dd48..dcc1a0a6f 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -132,7 +132,7 @@ object IvyXml { configurations0.toMap, None, Nil, - Map.empty, + Nil, Nil, None, None, diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 433d90c84..8e357e303 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -229,7 +229,7 @@ object Pom { Map.empty, parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), depMgmts, - properties.toMap, + properties, profiles, None, None, From 76e989d21c043e2eb4ab662029a4a22f28cfb264 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:46 +0100 Subject: [PATCH 63/80] Better properties substitution (#2) --- .../main/scala/coursier/core/Resolution.scala | 53 ++++++++++++++----- plugin/src/main/scala/coursier/FromSbt.scala | 2 +- .../scala/coursier/test/ResolutionTests.scala | 42 ++++++--------- .../test/scala/coursier/test/package.scala | 2 +- 4 files changed, 57 insertions(+), 42 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index 5db05a72d..ac197caa4 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -613,20 +613,25 @@ case class Resolution( .map(_._2.properties) .fold(project.properties)(project.properties ++ _) - val approxProperties = propertiesMap(approxProperties0) + val approxProperties = propertiesMap(approxProperties0) ++ Seq( + "project.groupId" -> project.module.organization, + "project.artifactId" -> project.module.name, + "project.version" -> project.version + ) val profileDependencies = profiles( project, approxProperties, profileActivation getOrElse defaultProfileActivation - ).flatMap(_.dependencies) + ).flatMap(p => p.dependencies ++ p.dependencyManagement) - val modules = - (project.dependencies ++ profileDependencies) - .collect{ - case ("import", dep) => dep.moduleVersion - } + val modules = withProperties( + project.dependencies ++ project.dependencyManagement ++ profileDependencies, + approxProperties + ).collect { + case ("import", dep) => dep.moduleVersion + } modules.toSet ++ project.parent } @@ -683,13 +688,21 @@ case class Resolution( */ def withDependencyManagement(project: Project): Project = { + // A bit fragile, but seems to work + // TODO Add non regression test for the touchy org.glassfish.jersey.core:jersey-client:2.19 + // (for the way it uses org.glassfish.hk2:hk2-bom,2.4.0-b25) + val approxProperties0 = project.parent .filter(projectCache.contains) .map(projectCache(_)._2.properties.toMap) .fold(project.properties)(project.properties ++ _) - val approxProperties = propertiesMap(approxProperties0) + val approxProperties = propertiesMap(approxProperties0) ++ Seq( + "project.groupId" -> project.module.organization, + "project.artifactId" -> project.module.name, + "project.version" -> project.version + ) val profiles0 = profiles( project, @@ -698,20 +711,29 @@ case class Resolution( ) val dependencies0 = addDependencies( - project.dependencies +: profiles0.map(_.dependencies) + (project.dependencies +: profiles0.map(_.dependencies)).map(withProperties(_, approxProperties)) + ) + val dependenciesMgmt0 = addDependencies( + (project.dependencyManagement +: profiles0.map(_.dependencyManagement)).map(withProperties(_, approxProperties)) ) val properties0 = (project.properties /: profiles0) { (acc, p) => acc ++ p.properties } - val deps = ( + val deps0 = ( dependencies0 .collect { case ("import", dep) => dep.moduleVersion } ++ + dependenciesMgmt0 + .collect { case ("import", dep) => + dep.moduleVersion + } ++ project.parent - ).filter(projectCache.contains) + ) + + val deps = deps0.filter(projectCache.contains) val projs = deps .map(projectCache(_)._2) @@ -721,7 +743,9 @@ case class Resolution( profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement) ) - ).foldLeft(Map.empty[DepMgmt.Key, (String, Dependency)])(DepMgmt.addSeq) + ) + .map(withProperties(_, approxProperties)) + .foldLeft(Map.empty[DepMgmt.Key, (String, Dependency)])(DepMgmt.addSeq) val depsSet = deps.toSet @@ -735,7 +759,10 @@ case class Resolution( .filter(projectCache.contains) .toSeq .flatMap(projectCache(_)._2.dependencies), - dependencyManagement = depMgmt.values.toSeq, + dependencyManagement = depMgmt.values.toSeq + .filterNot{case (config, dep) => + config == "import" && depsSet(dep.moduleVersion) + }, properties = project.parent .filter(projectCache.contains) .map(projectCache(_)._2.properties) diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 0611e2ed2..0bbc0349b 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -105,7 +105,7 @@ object FromSbt { ivyConfigurations, None, Nil, - Map.empty, + Nil, Nil, None, None, diff --git a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala index 123461fc1..e8dab1420 100644 --- a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala +++ b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala @@ -36,7 +36,7 @@ object ResolutionTests extends TestSuite { dependencies = Seq( "" -> Dependency(Module("acme", "play-json"), "${playJsonVersion}"), "" -> Dependency(Module("${project.groupId}", "${configName}"), "1.3.0")), - properties = Map( + properties = Seq( "playJsonVersion" -> "2.4.0", "configName" -> "config")), @@ -83,7 +83,7 @@ object ResolutionTests extends TestSuite { dependencies = Seq( "" -> Dependency(Module("gov.nsa", "secure-pgp"), "10.0", exclusions = Set(("*", "${crypto.name}")))), - properties = Map("crypto.name" -> "crypto", "dummy" -> "2")), + properties = Seq("crypto.name" -> "crypto", "dummy" -> "2")), Project(Module("com.thoughtworks.paranamer", "paranamer-parent"), "2.6", dependencies = Seq( @@ -108,29 +108,29 @@ object ResolutionTests extends TestSuite { "test" -> Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4"))))), Project(Module("com.github.dummy", "libb"), "0.5.3", - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> None)), dependencies = Seq( "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.5.4", - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("true"))), dependencies = Seq( "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.5.5", - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq( "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb-parent"), "0.5.6", - properties = Map("special" -> "true")), + properties = Seq("special" -> "true")), Project(Module("com.github.dummy", "libb"), "0.5.6", parent = Some(Module("com.github.dummy", "libb-parent"), "0.5.6"), - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq( "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), @@ -243,14 +243,11 @@ object ResolutionTests extends TestSuite { ) val res = await(resolve0( Set(dep) - )) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) + dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) assert(res == expected) @@ -267,14 +264,11 @@ object ResolutionTests extends TestSuite { ) val res = await(resolve0( Set(dep) - )) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) + dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) assert(res == expected) @@ -291,14 +285,11 @@ object ResolutionTests extends TestSuite { ) val res = await(resolve0( Set(dep) - )) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) + dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) assert(res == expected) @@ -309,14 +300,11 @@ object ResolutionTests extends TestSuite { val dep = Dependency(Module("hudsucker", "mail"), "10.0") val res = await(resolve0( Set(dep) - )).copy(filter = None) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) + dependencies = Set(dep.withCompileScope) ) assert(res == expected) diff --git a/tests/shared/src/test/scala/coursier/test/package.scala b/tests/shared/src/test/scala/coursier/test/package.scala index 1bb6d4487..30fa8f3ea 100644 --- a/tests/shared/src/test/scala/coursier/test/package.scala +++ b/tests/shared/src/test/scala/coursier/test/package.scala @@ -39,7 +39,7 @@ package object test { parent: Option[ModuleVersion] = None, dependencyManagement: Seq[(String, Dependency)] = Seq.empty, configurations: Map[String, Seq[String]] = Map.empty, - properties: Map[String, String] = Map.empty, + properties: Seq[(String, String)] = Seq.empty, profiles: Seq[Profile] = Seq.empty, versions: Option[core.Versions] = None, snapshotVersioning: Option[core.SnapshotVersioning] = None, From 403b3ddf7dd7b4a97bcc78d373e4d966dc7ae153 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:46 +0100 Subject: [PATCH 64/80] Fix Required to compile the readme module of Ammonite Surprised that this one wasn't caught earlier --- plugin/src/main/scala/coursier/InterProjectRepository.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/main/scala/coursier/InterProjectRepository.scala b/plugin/src/main/scala/coursier/InterProjectRepository.scala index 3c87631e5..9e21bb58d 100644 --- a/plugin/src/main/scala/coursier/InterProjectRepository.scala +++ b/plugin/src/main/scala/coursier/InterProjectRepository.scala @@ -30,7 +30,7 @@ case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artif projects.map { case (proj, artifactsByConfig) => val artifacts = artifactsByConfig.toMap val allArtifacts = proj.allConfigurations.map { case (config, extends0) => - config -> extends0.toSeq.flatMap(artifacts.getOrElse(_, Nil)) + config -> (extends0 + config).toSeq.flatMap(artifacts.getOrElse(_, Nil)) } proj.moduleVersion -> allArtifacts }.toMap From 2e6e061fe1a1b00dbaf1851340587d1a2d782039 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:47 +0100 Subject: [PATCH 65/80] Better minimization of dependencies with fallback configs --- .../shared/src/main/scala/coursier/core/Orders.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Orders.scala b/core/shared/src/main/scala/coursier/core/Orders.scala index d2bd8cdb7..15af6bfe2 100644 --- a/core/shared/src/main/scala/coursier/core/Orders.scala +++ b/core/shared/src/main/scala/coursier/core/Orders.scala @@ -122,8 +122,16 @@ object Orders { private def fallbackConfigIfNecessary(dep: Dependency, configs: Set[String]): Dependency = Parse.withFallbackConfig(dep.configuration) match { - case Some((main, fallback)) if !configs(main) && configs(fallback) => - dep.copy(configuration = fallback) + case Some((main, fallback)) => + val config0 = + if (configs(main)) + main + else if (configs(fallback)) + fallback + else + dep.configuration + + dep.copy(configuration = config0) case _ => dep } From 357c3fa0f79d1f5bcae2ecac2cf8209fe56ea99e Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:47 +0100 Subject: [PATCH 66/80] Fix parsing of dependencies config mapping of Ivy files --- .../src/main/scala/coursier/ivy/IvyXml.scala | 18 +++++++++++++++--- plugin/src/main/scala/coursier/FromSbt.scala | 17 ++--------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala index dcc1a0a6f..5f15e255d 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -28,6 +28,20 @@ object IvyXml { name -> node.attribute("extends").toOption.toSeq.flatMap(_.split(',')) } + // FIXME "default(compile)" likely not to be always the default + def mappings(mapping: String): Seq[(String, String)] = + mapping.split(';').flatMap { m => + val (froms, tos) = m.split("->", 2) match { + case Array(from) => (from, "default(compile)") + case Array(from, to) => (from, to) + } + + for { + from <- froms.split(',') + to <- tos.split(',') + } yield (from.trim, to.trim) + } + // FIXME Errors ignored as above - warnings should be reported at least for anything suspicious private def dependencies(node: Node): Seq[(String, Dependency)] = node.children @@ -53,9 +67,7 @@ object IvyXml { name <- node.attribute("name").toOption.toSeq version <- node.attribute("rev").toOption.toSeq rawConf <- node.attribute("conf").toOption.toSeq - (fromConf, toConf) <- rawConf.split(',').toSeq.map(_.split("->", 2)).collect { - case Array(from, to) => from -> to - } + (fromConf, toConf) <- mappings(rawConf) attr = node.attributesFromNamespace(attributesNamespace) } yield fromConf -> Dependency( Module(org, name, attr.toMap), diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala index 0bbc0349b..2a1b653b1 100644 --- a/plugin/src/main/scala/coursier/FromSbt.scala +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -1,6 +1,6 @@ package coursier -import coursier.ivy.IvyRepository +import coursier.ivy.{ IvyXml, IvyRepository } import sbt.mavenint.SbtPomExtraProperties import sbt.{ Resolver, CrossVersion, ModuleID } @@ -24,19 +24,6 @@ object FromSbt { case f: CrossVersion.Binary => name + "_" + f.remapVersion(scalaBinaryVersion) } - def mappings(mapping: String): Seq[(String, String)] = - mapping.split(';').flatMap { m => - val (froms, tos) = m.split("->", 2) match { - case Array(from) => (from, "default(compile)") - case Array(from, to) => (from, to) - } - - for { - from <- froms.split(',') - to <- tos.split(',') - } yield (from.trim, to.trim) - } - def attributes(attr: Map[String, String]): Map[String, String] = attr.map { case (k, v) => k.stripPrefix("e:") -> v @@ -64,7 +51,7 @@ object FromSbt { ) val mapping = module.configurations.getOrElse("compile") - val allMappings = mappings(mapping) + val allMappings = IvyXml.mappings(mapping) val attributes = if (module.explicitArtifacts.isEmpty) From 65a0c4ae5537ede7cd5e6e9d122b0aca460033e2 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:47 +0100 Subject: [PATCH 67/80] Print minimized dependencies in plugin output --- plugin/src/main/scala/coursier/Tasks.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index 4ac4eed3f..bc103b52a 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -305,7 +305,7 @@ object Tasks { if (verbosity >= 0) errPrintln("Resolution done") if (verbosity >= 1) - for (depRepr <- depsRepr0(res.dependencies.toSeq)) + for (depRepr <- depsRepr0(res.minDependencies.toSeq)) errPrintln(s" $depRepr") def repr(dep: Dependency) = { From 66fff86f9b2cc2dfed13687fb15544c884a73684 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:47 +0100 Subject: [PATCH 68/80] Don't try to download local files Regression... --- cache/src/main/scala/coursier/Cache.scala | 34 +++++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index af82d2da3..e2026d3a2 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -224,21 +224,31 @@ object Cache { for ((f, url) <- pairs) yield { val file = new File(f) - val res = cachePolicy match { - case CachePolicy.LocalOnly => + val res = + if (url.startsWith("file:/")) { + assert( + url.stripPrefix("file:/").stripPrefix("//") == + file.toURI.toString.stripPrefix("file:/").stripPrefix("//"), + s"URL: ${url.stripPrefix("file:/").stripPrefix("//")}, " + + s"file: ${file.toURI.toString.stripPrefix("file:/").stripPrefix("//")}" + ) checkFileExists(file, url) - case CachePolicy.UpdateChanging | CachePolicy.Update => - shouldDownload(file, url).flatMap { - case true => + } else + cachePolicy match { + case CachePolicy.LocalOnly => + checkFileExists(file, url) + case CachePolicy.UpdateChanging | CachePolicy.Update => + shouldDownload(file, url).flatMap { + case true => + remote(file, url) + case false => + EitherT(Task.now(\/-(()) : FileError \/ Unit)) + } + case CachePolicy.FetchMissing => + checkFileExists(file, url) orElse remote(file, url) + case CachePolicy.ForceDownload => remote(file, url) - case false => - EitherT(Task.now(\/-(()) : FileError \/ Unit)) } - case CachePolicy.FetchMissing => - checkFileExists(file, url) orElse remote(file, url) - case CachePolicy.ForceDownload => - remote(file, url) - } res.run.map((file, url) -> _) } From 139525355afe45ea6897b454f17623346bbe27e1 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:48 +0100 Subject: [PATCH 69/80] Better terminal output --- .../src/main/scala/coursier/TermDisplay.scala | 13 +++++++++++- cli/src/main/scala/coursier/cli/Helper.scala | 20 ++++++++++--------- plugin/src/main/scala/coursier/Tasks.scala | 20 ++++++++++++++----- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index f78cf7200..4447731db 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -19,6 +19,7 @@ class TermDisplay( private val fallbackRefreshInterval = 1000 private val lock = new AnyRef + private var currentHeight = 0 private val t = new Thread("TermDisplay") { override def run() = lock.synchronized { @@ -60,7 +61,9 @@ class TermDisplay( } - @tailrec def helper(lineCount: Int): Unit = + @tailrec def helper(lineCount: Int): Unit = { + currentHeight = lineCount + Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { case None => helper(lineCount) case Some(Left(())) => // poison pill @@ -100,6 +103,7 @@ class TermDisplay( Thread.sleep(refreshInterval) helper(downloads0.length) } + } @tailrec def fallbackHelper(previous: Set[String]): Unit = @@ -153,6 +157,13 @@ class TermDisplay( } def stop(): Unit = { + for (_ <- 0 until currentHeight) { + ansi.clearLine(2) + ansi.down(1) + } + for (_ <- 0 until currentHeight) { + ansi.up(1) + } q.put(Left(())) lock.synchronized(()) } diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 48b01a276..7394c764c 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -213,7 +213,6 @@ class Helper( Some(new TermDisplay(new OutputStreamWriter(System.err))) else None - logger.foreach(_.init()) val fetchs = cachePolicies.map(p => Cache.fetch(caches, p, logger = logger, pool = pool) @@ -246,6 +245,7 @@ class Helper( } } + logger.foreach(_.init()) val res = startRes .process @@ -341,24 +341,26 @@ class Helper( Some(new TermDisplay(new OutputStreamWriter(System.err))) else None - logger.foreach(_.init()) + + if (verbose0 >= 1 && artifacts.nonEmpty) + println(s"Found ${artifacts.length} artifacts") + val tasks = artifacts.map(artifact => (Cache.file(artifact, caches, cachePolicies.head, logger = logger, pool = pool) /: cachePolicies.tail)( _ orElse Cache.file(artifact, caches, _, logger = logger, pool = pool) ).run.map(artifact.->) ) - def printTask = Task { - if (verbose0 >= 1 && artifacts.nonEmpty) - println(s"Found ${artifacts.length} artifacts") - } - val task = printTask.flatMap(_ => Task.gatherUnordered(tasks)) + + logger.foreach(_.init()) + + val task = Task.gatherUnordered(tasks) + + logger.foreach(_.stop()) val results = task.run val errors = results.collect{case (artifact, -\/(err)) => artifact -> err } val files0 = results.collect{case (artifact, \/-(f)) => f } - logger.foreach(_.stop()) - if (errors.nonEmpty) { println(s"${errors.size} error(s):") for ((artifact, error) <- errors) { diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index bc103b52a..dc0e97c48 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -251,16 +251,17 @@ object Tasks { val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) - val logger = new TermDisplay( + def createLogger() = new TermDisplay( new OutputStreamWriter(System.err), fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty ) - logger.init() + + val resLogger = createLogger() val fetch = coursier.Fetch( repositories, - Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(logger), pool = pool), - Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(logger), pool = pool) + Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool), + Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool) ) def depsRepr(deps: Seq[(String, Dependency)]) = @@ -290,6 +291,8 @@ object Tasks { for (depRepr <- depsRepr(currentProject.dependencies)) errPrintln(s" $depRepr") + resLogger.init() + val res = startRes .process .run(fetch, maxIterations) @@ -297,6 +300,7 @@ object Tasks { .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) .merge + resLogger.stop() if (!res.isDone) @@ -366,13 +370,17 @@ object Tasks { case Some(cl) => res.classifiersArtifacts(cl) } + val artifactsLogger = createLogger() + val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(logger), pool = pool).run.map((a, _)) + Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _)) } if (verbosity >= 0) errPrintln(s"Fetching artifacts") + artifactsLogger.init() + val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { case -\/(ex) => throw new Exception(s"Error while downloading / verifying artifacts", ex) @@ -380,6 +388,8 @@ object Tasks { l.toMap } + artifactsLogger.stop() + if (verbosity >= 0) errPrintln(s"Fetching artifacts: done") From bfc9a10639ea90f5e416b10f438c7f40e31faa3f Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:48 +0100 Subject: [PATCH 70/80] Fix up... --- cache/src/main/scala/coursier/Cache.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index e2026d3a2..ee057c4f7 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -226,11 +226,11 @@ object Cache { val res = if (url.startsWith("file:/")) { + def filtered(s: String) = + s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/") assert( - url.stripPrefix("file:/").stripPrefix("//") == - file.toURI.toString.stripPrefix("file:/").stripPrefix("//"), - s"URL: ${url.stripPrefix("file:/").stripPrefix("//")}, " + - s"file: ${file.toURI.toString.stripPrefix("file:/").stripPrefix("//")}" + filtered(url) == filtered(file.toURI.toString), + s"URL: ${filtered(url)}, file: ${filtered(file.toURI.toString)}" ) checkFileExists(file, url) } else From 8540ba3078dd13b2a9e2f6797d2e2db6755d40e6 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:48 +0100 Subject: [PATCH 71/80] No need to supply pseudo-artifacts for sub-projects --- .../coursier/InterProjectRepository.scala | 34 ++----------------- plugin/src/main/scala/coursier/Keys.scala | 4 +-- plugin/src/main/scala/coursier/Tasks.scala | 30 ++++------------ 3 files changed, 11 insertions(+), 57 deletions(-) diff --git a/plugin/src/main/scala/coursier/InterProjectRepository.scala b/plugin/src/main/scala/coursier/InterProjectRepository.scala index 9e21bb58d..8eb094420 100644 --- a/plugin/src/main/scala/coursier/InterProjectRepository.scala +++ b/plugin/src/main/scala/coursier/InterProjectRepository.scala @@ -2,40 +2,12 @@ package coursier import scalaz.{ -\/, \/-, Monad, EitherT } -case class InterProjectSource(artifacts: Map[(Module, String), Map[String, Seq[Artifact]]]) extends Artifact.Source { - def artifacts( - dependency: Dependency, - project: Project, - overrideClassifiers: Option[Seq[String]] - ): Seq[Artifact] = - overrideClassifiers match { - case None => - artifacts - .get(dependency.moduleVersion) - .toSeq - .flatMap(_.get(dependency.configuration)) - .flatten - case Some(_) => - Nil - } -} - -case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artifact])])]) extends Repository { +case class InterProjectRepository(projects: Seq[Project]) extends Repository { private val map = projects - .map { case (proj, _) => proj.moduleVersion -> proj } + .map { proj => proj.moduleVersion -> proj } .toMap - val source = InterProjectSource( - projects.map { case (proj, artifactsByConfig) => - val artifacts = artifactsByConfig.toMap - val allArtifacts = proj.allConfigurations.map { case (config, extends0) => - config -> (extends0 + config).toSeq.flatMap(artifacts.getOrElse(_, Nil)) - } - proj.moduleVersion -> allArtifacts - }.toMap - ) - def find[F[_]]( module: Module, version: String, @@ -45,7 +17,7 @@ case class InterProjectRepository(projects: Seq[(Project, Seq[(String, Seq[Artif ): EitherT[F, String, (Artifact.Source, Project)] = { val res = map.get((module, version)) match { case Some(proj) => - \/-((source, proj)) + \/-((Artifact.Source.empty, proj)) case None => -\/("Not found") } diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala index 3ae648fe3..d870f320d 100644 --- a/plugin/src/main/scala/coursier/Keys.scala +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -18,8 +18,8 @@ object Keys { val coursierCache = SettingKey[File]("coursier-cache", "") - val coursierProject = TaskKey[(Project, Seq[(String, Seq[Artifact])])]("coursier-project", "") - val coursierProjects = TaskKey[Seq[(Project, Seq[(String, Seq[Artifact])])]]("coursier-projects", "") + val coursierProject = TaskKey[Project]("coursier-project", "") + val coursierProjects = TaskKey[Seq[Project]]("coursier-projects", "") val coursierPublications = TaskKey[Seq[(String, Publication)]]("coursier-publications", "") val coursierSbtClassifiersModule = TaskKey[GetClassifiersModule]("coursier-sbt-classifiers-module", "") diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala index dc0e97c48..1e43da038 100644 --- a/plugin/src/main/scala/coursier/Tasks.scala +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -32,7 +32,7 @@ object Tasks { resolvers } - def coursierProjectTask: Def.Initialize[sbt.Task[(Project, Seq[(String, Seq[Artifact])])]] = + def coursierProjectTask: Def.Initialize[sbt.Task[Project]] = ( sbt.Keys.state, sbt.Keys.thisProjectRef @@ -41,41 +41,23 @@ object Tasks { // should projectID.configurations be used instead? val configurations = ivyConfigurations.in(projectRef).get(state) - // exportedProducts looks like what we want, but depends on the update task, which - // make the whole thing run into cycles... - val artifacts = configurations.map { cfg => - cfg.name -> Option(classDirectory.in(projectRef).in(cfg).getOrElse(state, null)) - }.collect { case (name, Some(classDir)) => - name -> Seq( - Artifact( - classDir.toURI.toString, - Map.empty, - Map.empty, - Attributes(), - changing = true - ) - ) - } - val allDependenciesTask = allDependencies.in(projectRef).get(state) for { allDependencies <- allDependenciesTask } yield { - val proj = FromSbt.project( + FromSbt.project( projectID.in(projectRef).get(state), allDependencies, configurations.map { cfg => cfg.name -> cfg.extendsConfigs.map(_.name) }.toMap, scalaVersion.in(projectRef).get(state), scalaBinaryVersion.in(projectRef).get(state) ) - - (proj, artifacts) } } - def coursierProjectsTask: Def.Initialize[sbt.Task[Seq[(Project, Seq[(String, Seq[Artifact])])]]] = + def coursierProjectsTask: Def.Initialize[sbt.Task[Seq[Project]]] = sbt.Keys.state.flatMap { state => val projects = structure(state).allProjectRefs coursierProject.forAllProjects(state, projects).map(_.values.toVector) @@ -162,7 +144,7 @@ object Tasks { scalaBinaryVersion.value ) else { - val (proj, _) = coursierProject.value + val proj = coursierProject.value val publications = coursierPublications.value proj.copy(publications = publications) } @@ -202,7 +184,7 @@ object Tasks { val startRes = Resolution( currentProject.dependencies.map { case (_, dep) => dep }.toSet, filter = Some(dep => !dep.optional), - forceVersions = projects.map { case (proj, _) => proj.moduleVersion }.toMap + forceVersions = projects.map(_.moduleVersion).toMap ) // required for publish to be fine, later on @@ -224,7 +206,7 @@ object Tasks { def report = { if (verbosity >= 1) { println("InterProjectRepository") - for ((p, _) <- projects) + for (p <- projects) println(s" ${p.module}:${p.version}") } From a1db5f1fdc4f8231a6cc86ab8794e8696cedb8df Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:48 +0100 Subject: [PATCH 72/80] Better locks in cache --- cache/src/main/scala/coursier/Cache.scala | 178 ++++++++++++++-------- 1 file changed, 113 insertions(+), 65 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index ee057c4f7..8d05a746e 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -2,6 +2,7 @@ package coursier import java.net.{HttpURLConnection, URL} import java.nio.channels.{ OverlappingFileLockException, FileLock } +import java.nio.file.{ StandardCopyOption, Files => NioFiles } import java.security.MessageDigest import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService} @@ -48,6 +49,98 @@ object Cache { )) } + private def readFullyTo( + in: InputStream, + out: OutputStream, + logger: Option[Logger], + url: String + ): Unit = { + + val b = Array.fill[Byte](bufferSize)(0) + + @tailrec + def helper(count: Long): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + out.flush() + logger.foreach(_.downloadProgress(url, count + read)) + helper(count + read) + } + } + + helper(0L) + } + + private def withLockFor[T](file: File)(f: => FileError \/ T): FileError \/ T = { + val lockFile = new File(file.getParentFile, s"${file.getName}.lock") + + lockFile.getParentFile.mkdirs() + var out = new FileOutputStream(lockFile) + + try { + var lock: FileLock = null + try { + lock = out.getChannel.tryLock() + if (lock == null) + -\/(FileError.Locked(file)) + else + try f + finally { + lock.release() + lock = null + out.close() + out = null + lockFile.delete() + } + } + catch { + case e: OverlappingFileLockException => + -\/(FileError.Locked(file)) + } + finally if (lock != null) lock.release() + } finally if (out != null) out.close() + } + + private def downloading[T]( + url: String, + file: File, + logger: Option[Logger] + )( + f: => FileError \/ T + ): FileError \/ T = + try { + val o = new Object + val prev = urlLocks.putIfAbsent(url, o) + if (prev == null) { + logger.foreach(_.downloadingArtifact(url, file)) + + val res = + try f + catch { case e: Exception => + logger.foreach(_.downloadedArtifact(url, success = false)) + throw e + } + finally { + urlLocks.remove(url) + } + + logger.foreach(_.downloadedArtifact(url, success = true)) + + res + } else + -\/(FileError.ConcurrentDownload(url)) + } + catch { case e: Exception => + -\/(FileError.DownloadError(e.getMessage)) + } + + private def temporaryFile(file: File): File = { + val dir = file.getParentFile + val name = file.getName + new File(dir, s"$name.part") + } + private def download( artifact: Artifact, cache: Seq[(String, File)], @@ -130,81 +223,36 @@ object Cache { fromDatesOpt.getOrElse(true) } - // FIXME Things can go wrong here and are not properly handled, - // e.g. what if the connection gets closed during the transfer? - // (partial file on disk?) def remote(file: File, url: String): EitherT[Task, FileError, Unit] = EitherT { Task { - try { - val o = new Object - val prev = urlLocks.putIfAbsent(url, o) - if (prev == null) { - logger.foreach(_.downloadingArtifact(url, file)) + withLockFor(file) { + downloading(url, file, logger) { + val conn = urlConn(url) - val r = try { - val conn = urlConn(url) + for (len <- Option(conn.getContentLengthLong) if len >= 0L) + logger.foreach(_.downloadLength(url, len)) - for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) - logger.foreach(_.downloadLength(url, len)) + val in = new BufferedInputStream(conn.getInputStream, bufferSize) - val in = new BufferedInputStream(conn.getInputStream, bufferSize) + val tmp = temporaryFile(file) - val result = - try { - file.getParentFile.mkdirs() - val out = new FileOutputStream(file) - try { - var lock: FileLock = null - try { - lock = out.getChannel.tryLock() - if (lock == null) - -\/(FileError.Locked(file)) - else { - val b = Array.fill[Byte](bufferSize)(0) + val result = + try { + tmp.getParentFile.mkdirs() + val out = new FileOutputStream(tmp) + try \/-(readFullyTo(in, out, logger, url)) + finally out.close() + } finally in.close() - @tailrec - def helper(count: Long): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - out.flush() - logger.foreach(_.downloadProgress(url, count + read)) - helper(count + read) - } - } + file.getParentFile.mkdirs() + NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE) - helper(0L) - \/-(()) - } - } - catch { - case e: OverlappingFileLockException => - -\/(FileError.Locked(file)) - } - finally if (lock != null) lock.release() - } finally out.close() - } finally in.close() + for (lastModified <- Option(conn.getLastModified) if lastModified > 0L) + file.setLastModified(lastModified) - for (lastModified <- Option(conn.getLastModified).filter(_ > 0L)) - file.setLastModified(lastModified) - - result - } - catch { case e: Exception => - logger.foreach(_.downloadedArtifact(url, success = false)) - throw e - } - finally { - urlLocks.remove(url) - } - logger.foreach(_.downloadedArtifact(url, success = true)) - r - } else - -\/(FileError.ConcurrentDownload(url)) - } - catch { case e: Exception => - -\/(FileError.DownloadError(e.getMessage)) + result + } } } } From 2463e41df17526c5db3cfaf99b3ca7288e544ddd Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:49 +0100 Subject: [PATCH 73/80] Fix up (...) --- cli/src/main/scala/coursier/cli/Helper.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 7394c764c..5bc66ff75 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -355,12 +355,12 @@ class Helper( val task = Task.gatherUnordered(tasks) - logger.foreach(_.stop()) - val results = task.run val errors = results.collect{case (artifact, -\/(err)) => artifact -> err } val files0 = results.collect{case (artifact, \/-(f)) => f } + logger.foreach(_.stop()) + if (errors.nonEmpty) { println(s"${errors.size} error(s):") for ((artifact, error) <- errors) { From 7a8a626064df27bfcc3968efd9fb6b471cf097f0 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:49 +0100 Subject: [PATCH 74/80] Resume partial downloads --- cache/src/main/scala/coursier/Cache.scala | 41 ++++++++++++++++++----- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 8d05a746e..49086730a 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -53,7 +53,8 @@ object Cache { in: InputStream, out: OutputStream, logger: Option[Logger], - url: String + url: String, + alreadyDownloaded: Long ): Unit = { val b = Array.fill[Byte](bufferSize)(0) @@ -69,7 +70,7 @@ object Cache { } } - helper(0L) + helper(alreadyDownloaded) } private def withLockFor[T](file: File)(f: => FileError \/ T): FileError \/ T = { @@ -141,6 +142,8 @@ object Cache { new File(dir, s"$name.part") } + private val partialContentResponseCode = 206 + private def download( artifact: Artifact, cache: Seq[(String, File)], @@ -228,20 +231,42 @@ object Cache { Task { withLockFor(file) { downloading(url, file, logger) { - val conn = urlConn(url) + val tmp = temporaryFile(file) - for (len <- Option(conn.getContentLengthLong) if len >= 0L) + val alreadyDownloaded = tmp.length() + + val conn0 = urlConn(url) + + val (partialDownload, conn) = conn0 match { + case conn0: HttpURLConnection if alreadyDownloaded > 0L => + conn0.setRequestProperty("Range", s"bytes=$alreadyDownloaded-") + + if (conn0.getResponseCode == partialContentResponseCode) { + val ackRange = Option(conn0.getHeaderField("Content-Range")).getOrElse("") + + if (ackRange.startsWith(s"bytes $alreadyDownloaded-")) + (true, conn0) + else + // unrecognized Content-Range header -> start a new connection with no resume + (false, urlConn(url)) + } else + (false, conn0) + + case _ => (false, conn0) + } + + for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) { + val len = len0 + (if (partialDownload) alreadyDownloaded else 0L) logger.foreach(_.downloadLength(url, len)) + } val in = new BufferedInputStream(conn.getInputStream, bufferSize) - val tmp = temporaryFile(file) - val result = try { tmp.getParentFile.mkdirs() - val out = new FileOutputStream(tmp) - try \/-(readFullyTo(in, out, logger, url)) + val out = new FileOutputStream(tmp, partialDownload) + try \/-(readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)) finally out.close() } finally in.close() From 340add13e7946b2fa2883c8a4677e214642251ba Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:49 +0100 Subject: [PATCH 75/80] Better error message --- cache/src/main/scala/coursier/Cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 49086730a..e66bda7e1 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -133,7 +133,7 @@ object Cache { -\/(FileError.ConcurrentDownload(url)) } catch { case e: Exception => - -\/(FileError.DownloadError(e.getMessage)) + -\/(FileError.DownloadError(s"Caught $e (${e.getMessage})")) } private def temporaryFile(file: File): File = { From e7b57af58b42fb61e0159e937463881d153386c9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 01:34:50 +0100 Subject: [PATCH 76/80] Add extra properties --- .../shared/src/main/scala/coursier/core/Resolution.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index ac197caa4..bd534e015 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -324,7 +324,14 @@ object Resolution { // come from parents or dependency management. This may not be // the right thing to do. - val properties0 = project.properties ++ Seq( + // FIXME The extra properties should only be added for Maven projects, not Ivy ones + val properties0 = Seq( + // some artifacts seem to require these (e.g. org.jmock:jmock-legacy:2.5.1) + // although I can find no mention of them in any manual / spec + "pom.groupId" -> project.module.organization, + "pom.artifactId" -> project.module.name, + "pom.version" -> project.version + ) ++ project.properties ++ Seq( "project.groupId" -> project.module.organization, "project.artifactId" -> project.module.name, "project.version" -> project.version From c6d8c35b5de8a4c80fb7856064f121d67e342c6b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 20:43:33 +0100 Subject: [PATCH 77/80] Switch to case-app 1.0.0-M1 --- build.sbt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index ff9e10dc6..da18bde5f 100644 --- a/build.sbt +++ b/build.sbt @@ -51,8 +51,7 @@ lazy val baseCommonSettings = Seq( organization := "com.github.alexarchambault", resolvers ++= Seq( "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases", - Resolver.sonatypeRepo("releases"), - Resolver.sonatypeRepo("snapshots") + Resolver.sonatypeRepo("releases") ), scalacOptions += "-target:jvm-1.7", javacOptions ++= Seq( @@ -168,7 +167,7 @@ lazy val cli = project name := "coursier-cli", libraryDependencies ++= Seq( // beware - available only in 2.11 - "com.github.alexarchambault" %% "case-app" % "1.0.0-M1-SNAPSHOT", + "com.github.alexarchambault" %% "case-app" % "1.0.0-M1", "ch.qos.logback" % "logback-classic" % "1.1.3" ), resourceGenerators in Compile += packageBin.in(bootstrap).in(Compile).map { jar => From 5369913f1006d6372e4e170398c91e58eb4328a9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 21:00:10 +0100 Subject: [PATCH 78/80] Fix travis script --- project/travis.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/project/travis.sh b/project/travis.sh index 32116325c..0b0fddb5e 100755 --- a/project/travis.sh +++ b/project/travis.sh @@ -30,22 +30,20 @@ function isMasterOrDevelop() { # web sub-project doesn't compile in 2.10 (no scalajs-react) if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then IS_210=1 - SBT_COMMANDS="cli/compile" + SBT_COMMANDS="bootstrap/compile coreJVM/compile coreJS/compile cache/compile web/compile testsJVM/test testsJS/test" else IS_210=0 - SBT_COMMANDS="compile" + SBT_COMMANDS="compile test" fi # Required for ~/.ivy2/local repo tests ~/sbt coreJVM/publish-local -SBT_COMMANDS="$SBT_COMMANDS test" - # TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed PUSH_GHPAGES=0 if isNotPr && publish && isMaster; then - SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish files/publish" + SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish cache/publish" if [ "$IS_210" = 1 ]; then SBT_COMMANDS="$SBT_COMMANDS plugin/publish" else From 8ab731d93ff85eb999bf72bb2a4748feb3162a1d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 21:17:34 +0100 Subject: [PATCH 79/80] Fix untested things :-| Bad --- core/shared/src/main/scala/coursier/maven/Pom.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 8e357e303..7f42eeaab 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -195,8 +195,8 @@ object Pom { .flatMap(_.children) .filter(_.label == "license") .flatMap { n => - n.attribute("name").toOption.map { name => - (name, n.attribute("url").toOption) + text(n, "name", "License name").toOption.map { name => + (name, text(n, "url", "License URL").toOption) }.toSeq } @@ -207,9 +207,9 @@ object Pom { .filter(_.label == "developer") .map { n => for { - id <- n.attribute("id") - name <- n.attribute("name") - url <- n.attribute("url") + id <- text(n, "id", "Developer ID") + name <- text(n, "name", "Developer name") + url <- text(n, "url", "Developer URL") } yield Info.Developer(id, name, url) } .collect { From 4af9863757b97be615a78c289c201bd0d96b9559 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 30 Dec 2015 21:29:20 +0100 Subject: [PATCH 80/80] Update test --- tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala | 2 +- .../coursier_2.11/{0.1.0-SNAPSHOT => 1.0.0-SNAPSHOT} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/{0.1.0-SNAPSHOT => 1.0.0-SNAPSHOT} (75%) diff --git a/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala index 05aa0a712..db4006289 100644 --- a/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala @@ -9,7 +9,7 @@ object IvyLocalTests extends TestSuite { 'coursier{ // Assume this module (and the sub-projects it depends on) is published locally CentralTests.resolutionCheck( - Module("com.github.alexarchambault", "coursier_2.11"), "0.1.0-SNAPSHOT", + Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-SNAPSHOT", Some(Cache.ivy2Local)) } } diff --git a/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT similarity index 75% rename from tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT rename to tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT index cc4c8e9b1..58ba0a4fc 100644 --- a/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT +++ b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT @@ -1,4 +1,4 @@ -com.github.alexarchambault:coursier_2.11:jar:0.1.0-SNAPSHOT +com.github.alexarchambault:coursier_2.11:jar:1.0.0-SNAPSHOT org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4 org.scala-lang.modules:scala-xml_2.11:jar:1.0.4 org.scala-lang:scala-library:jar:2.11.7