From bf097fa018b4df9522377b3f8d85aafacc7a4fcd Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Mon, 7 Nov 2016 11:22:49 +0100 Subject: [PATCH 1/4] Rework profile activation - don't activate profiles activated by default if some user activated profiles are provided, and - accept OS / JDK conditions in activation --- build.sbt | 12 + .../main/scala-2.11/coursier/cli/Helper.scala | 4 +- .../main/scala/coursier/core/Activation.scala | 131 +++++++ .../scala/coursier/core/Definitions.scala | 3 - .../main/scala/coursier/core/Resolution.scala | 74 +++- .../src/main/scala/coursier/maven/Pom.scala | 20 +- .../src/main/scala/coursier/package.scala | 9 +- .../src/main/scala-2.10/coursier/Tasks.scala | 10 +- .../org.apache.spark/spark-core_2.11/1.3.1 | 13 - .../scala/coursier/test/ActivationTests.scala | 325 ++++++++++++++++++ .../scala/coursier/test/CentralTests.scala | 16 +- .../test/scala/coursier/test/package.scala | 4 +- 12 files changed, 567 insertions(+), 54 deletions(-) create mode 100644 core/shared/src/main/scala/coursier/core/Activation.scala create mode 100644 tests/shared/src/test/scala/coursier/test/ActivationTests.scala diff --git a/build.sbt b/build.sbt index cf93ef33d..e91b6d279 100644 --- a/build.sbt +++ b/build.sbt @@ -130,6 +130,18 @@ lazy val core = crossProject import com.typesafe.tools.mima.core._ Seq( + // Since 1.0.0-M15 + // reworked profile activation + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Resolution.copy"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Resolution.profileActivation"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Resolution.copyWithCache"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Resolution.this"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Activation.copy"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Activation.this"), + ProblemFilters.exclude[MissingTypesProblem]("coursier.core.Activation$"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Activation.apply"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Resolution.profiles"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Resolution.apply"), // Since 1.0.0-M13 // reworked VersionConstraint ProblemFilters.exclude[MissingClassProblem]("coursier.core.VersionConstraint$Interval"), diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index c08757f3a..f952ccaf2 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -285,7 +285,9 @@ class Helper( dependencies.toSet, forceVersions = forceVersions, filter = Some(dep => keepOptional || !dep.optional), - profileActivation = Some(core.Resolution.userProfileActivation(userEnabledProfiles)) + userActivations = + if (userEnabledProfiles.isEmpty) None + else Some(userEnabledProfiles.iterator.map(_ -> true).toMap) ) val loggerFallbackMode = diff --git a/core/shared/src/main/scala/coursier/core/Activation.scala b/core/shared/src/main/scala/coursier/core/Activation.scala new file mode 100644 index 000000000..bf10b82d5 --- /dev/null +++ b/core/shared/src/main/scala/coursier/core/Activation.scala @@ -0,0 +1,131 @@ +package coursier.core + +import scalaz.{-\/, \/, \/-} + +// Maven-specific +final case class Activation( + properties: Seq[(String, Option[String])], + os: Activation.Os, + jdk: Option[VersionInterval \/ Seq[Version]] +) { + + def isEmpty: Boolean = properties.isEmpty && os.isEmpty && jdk.isEmpty + + def isActive( + currentProperties: Map[String, String], + osInfo: Activation.Os, + jdkVersion: Option[Version] + ): Boolean = { + + def fromProperties = properties.forall { + case (name, valueOpt) => + if (name.startsWith("!")) + currentProperties.get(name.drop(1)).isEmpty + else + currentProperties.get(name).exists { v => + valueOpt.forall { reqValue => + if (reqValue.startsWith("!")) + v != reqValue.drop(1) + else + v == reqValue + } + } + } + + def fromOs = os.isActive(osInfo) + + def fromJdk = jdk.forall { + case -\/(itv) => + jdkVersion.exists(itv.contains) + case \/-(versions) => + jdkVersion.exists(versions.contains) + } + + !isEmpty && fromProperties && fromOs && fromJdk + } +} + +object Activation { + + case class Os( + arch: Option[String], + families: Set[String], + name: Option[String], + version: Option[String] // FIXME Could this be an interval? + ) { + def isEmpty: Boolean = + arch.isEmpty && families.isEmpty && name.isEmpty && version.isEmpty + + def isActive(osInfo: Os): Boolean = + arch.forall(osInfo.arch.toSeq.contains) && + families.forall { f => + if (Os.knownFamilies(f)) + osInfo.families.contains(f) + else + osInfo.name.exists(_.contains(f)) + } && + name.forall(osInfo.name.toSeq.contains) && + version.forall(osInfo.version.toSeq.contains) + } + + object Os { + val empty = Os(None, Set(), None, None) + + // below logic adapted from https://github.com/sonatype/plexus-utils/blob/f2beca21c75084986b49b3ab7b5f0f988021dcea/src/main/java/org/codehaus/plexus/util/Os.java + // brought in https://github.com/alexarchambault/coursier/issues/341 by @eboto + + private val standardFamilies = Set( + "windows", + "os/2", + "netware", + "mac", + "os/400", + "openvms" + ) + + private[Os] val knownFamilies = standardFamilies ++ Seq( + "dos", + "tandem", + "unix", + "win9x", + "z/os" + ) + + def families(name: String, pathSep: String): Set[String] = { + + var families = standardFamilies.filter(f => name.indexOf(f) >= 0) + + if (pathSep == ";" && name.indexOf("netware") < 0) + families += "dos" + + if (name.indexOf("nonstop_kernel") >= 0) + families += "tandem" + + if (pathSep == ":" && name.indexOf("openvms") < 0 && (name.indexOf("mac") < 0 || name.endsWith("x"))) + families += "unix" + + if (name.indexOf("windows") >= 0 && (name.indexOf("95") >= 0 || name.indexOf("98") >= 0 || name.indexOf("me") >= 0 || name.indexOf("ce") >= 0)) + families += "win9x" + + if (name.indexOf("z/os") >= 0 || name.indexOf("os/390") >= 0) + families += "z/os" + + families + } + + def fromProperties(properties: Map[String, String]): Os = { + + val name = properties.get("os.name").map(_.toLowerCase) + + Os( + properties.get("os.arch").map(_.toLowerCase), + (for (n <- name; sep <- properties.get("path.separator")) + yield families(n, sep)).getOrElse(Set()), + name, + properties.get("os.version").map(_.toLowerCase) + ) + } + } + + val empty = Activation(Nil, Os.empty, None) +} \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 5446e936a..3a5fc6b46 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -127,9 +127,6 @@ object Info { val empty = Info("", "", Nil, Nil, None) } -// Maven-specific -final case class Activation(properties: Seq[(String, Option[String])]) - // Maven-specific final case class Profile( id: String, diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index f45c3bc4a..d7b6e6438 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -11,25 +11,53 @@ object Resolution { type ModuleVersion = (Module, String) + def profileIsActive( + profile: Profile, + properties: Map[String, String], + osInfo: Activation.Os, + jdkVersion: Option[Version], + userActivations: Option[Map[String, Boolean]] + ): Boolean = { + + val fromUserOrDefault = userActivations match { + case Some(activations) => + activations.get(profile.id) + case None => + if (profile.activeByDefault.toSeq.contains(true)) + Some(true) + else + None + } + + def fromActivation = profile.activation.isActive(properties, osInfo, jdkVersion) + + val res = fromUserOrDefault.getOrElse(fromActivation) + + // println(s"Profile\n$profile\n$res\n") + + res + } + /** * Get the active profiles of `project`, using the current properties `properties`, - * and `profileActivation` stating if a profile is active. + * and `profileActivations` stating if a profile is active. */ def profiles( project: Project, properties: Map[String, String], - profileActivation: (String, Activation, Map[String, String]) => Boolean - ): Seq[Profile] = { - - val activated = project.profiles - .filter(p => profileActivation(p.id, p.activation, properties)) - - def default = project.profiles - .filter(_.activeByDefault.toSeq.contains(true)) - - if (activated.isEmpty) default - else activated - } + osInfo: Activation.Os, + jdkVersion: Option[Version], + userActivations: Option[Map[String, Boolean]] + ): Seq[Profile] = + project.profiles.filter { profile => + profileIsActive( + profile, + properties, + osInfo, + jdkVersion, + userActivations + ) + } object DepMgmt { type Key = (String, String, String) @@ -500,7 +528,9 @@ final case class Resolution( errorCache: Map[Resolution.ModuleVersion, Seq[String]], finalDependenciesCache: Map[Dependency, Seq[Dependency]], filter: Option[Dependency => Boolean], - profileActivation: Option[(String, Activation, Map[String, String]) => Boolean] + osInfo: Activation.Os, + jdkVersion: Option[Version], + userActivations: Option[Map[String, Boolean]] ) { def copyWithCache( @@ -511,7 +541,9 @@ final case class Resolution( projectCache: Map[Resolution.ModuleVersion, (Artifact.Source, Project)] = projectCache, errorCache: Map[Resolution.ModuleVersion, Seq[String]] = errorCache, filter: Option[Dependency => Boolean] = filter, - profileActivation: Option[(String, Activation, Map[String, String]) => Boolean] = profileActivation + osInfo: Activation.Os = osInfo, + jdkVersion: Option[Version] = jdkVersion, + userActivations: Option[Map[String, Boolean]] = userActivations ): Resolution = copy( rootDependencies, @@ -522,7 +554,9 @@ final case class Resolution( errorCache, finalDependenciesCache ++ finalDependenciesCache0.asScala, filter, - profileActivation + osInfo, + jdkVersion, + userActivations ) import Resolution._ @@ -741,7 +775,9 @@ final case class Resolution( profiles( project, approxProperties, - profileActivation getOrElse defaultProfileActivation + osInfo, + jdkVersion, + userActivations ).flatMap(p => p.dependencies ++ p.dependencyManagement) val modules = withProperties( @@ -850,7 +886,9 @@ final case class Resolution( val profiles0 = profiles( project, approxProperties, - profileActivation getOrElse defaultProfileActivation + osInfo, + jdkVersion, + userActivations ) // 1.2 made from Pom.scala (TODO look at the very details?) diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 866045125..f24d85923 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -75,7 +75,23 @@ object Pom { } yield (name, valueOpt) } - (byDefault, Activation(properties)) + val osNodeOpt = node.children.collectFirst { case n if n.label == "os" => n } + + val os = Activation.Os( + osNodeOpt.flatMap(n => text(n, "arch", "").toOption), + osNodeOpt.flatMap(n => text(n, "family", "").toOption).toSet, + osNodeOpt.flatMap(n => text(n, "name", "").toOption), + osNodeOpt.flatMap(n => text(n, "version", "").toOption) + ) + + val jdk = text(node, "jdk", "").toOption.flatMap { s => + Parse.versionInterval(s).map(-\/(_)) + .orElse(Parse.version(s).map(v => \/-(Seq(v)))) + } + + val activation = Activation(properties, os, jdk) + + (byDefault, activation) } def profile(node: Node): String \/ Profile = { @@ -85,7 +101,7 @@ object Pom { val xmlActivationOpt = node.children .find(_.label == "activation") - val (activeByDefault, activation) = xmlActivationOpt.fold((Option.empty[Boolean], Activation(Nil)))(profileActivation) + val (activeByDefault, activation) = xmlActivationOpt.fold((Option.empty[Boolean], Activation.empty))(profileActivation) val xmlDeps = node.children .find(_.label == "dependencies") diff --git a/core/shared/src/main/scala/coursier/package.scala b/core/shared/src/main/scala/coursier/package.scala index 32a0adb63..20e767b62 100644 --- a/core/shared/src/main/scala/coursier/package.scala +++ b/core/shared/src/main/scala/coursier/package.scala @@ -1,3 +1,4 @@ +import coursier.core.{Activation, Parse, Version} /** * Mainly pulls definitions from coursier.core, sometimes with default arguments. @@ -72,7 +73,9 @@ package object coursier { errorCache: Map[ModuleVersion, Seq[String]] = Map.empty, finalDependencies: Map[Dependency, Seq[Dependency]] = Map.empty, filter: Option[Dependency => Boolean] = None, - profileActivation: Option[(String, core.Activation, Map[String, String]) => Boolean] = None + osInfo: Activation.Os = Activation.Os.fromProperties(sys.props.toMap), + jdkVersion: Option[Version] = sys.props.get("java.version").flatMap(Parse.version), + userActivations: Option[Map[String, Boolean]] = None ): Resolution = core.Resolution( rootDependencies, @@ -83,7 +86,9 @@ package object coursier { errorCache, finalDependencies, filter, - profileActivation + osInfo, + jdkVersion, + userActivations ) } diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index fd6316a32..c1d773d81 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -441,7 +441,11 @@ object Tasks { dep.copy(exclusions = dep.exclusions ++ exclusions) }.toSet, filter = Some(dep => !dep.optional), - profileActivation = Some(core.Resolution.userProfileActivation(userEnabledProfiles)), + userActivations = + if (userEnabledProfiles.isEmpty) + None + else + Some(userEnabledProfiles.iterator.map(_ -> true).toMap), forceVersions = // order matters here userForceVersions ++ @@ -677,7 +681,7 @@ object Tasks { currentProject, repositories, userEnabledProfiles, - startRes.copy(filter = None, profileActivation = None), + startRes.copy(filter = None), sbtClassifiers ), resolution @@ -910,7 +914,7 @@ object Tasks { reportsCache.getOrElseUpdate( ReportCacheKey( currentProject, - res.copy(filter = None, profileActivation = None), + res.copy(filter = None), withClassifiers, sbtClassifiers ), diff --git a/tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 b/tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 index b196ffc0a..5d5eca3b8 100644 --- a/tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 +++ b/tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 @@ -13,13 +13,9 @@ com.google.guava:guava:14.0.1:compile com.google.inject:guice:3.0:compile com.google.protobuf:protobuf-java:2.5.0:compile com.ning:compress-lzf:1.0.0:compile -com.sun.jersey:jersey-client:1.9:compile -com.sun.jersey:jersey-core:1.9:compile -com.sun.jersey:jersey-grizzly2:1.9:compile com.sun.jersey:jersey-json:1.9:compile com.sun.jersey:jersey-server:1.9:compile com.sun.jersey.contribs:jersey-guice:1.9:compile -com.sun.jersey.jersey-test-framework:jersey-test-framework-core:1.9:compile com.sun.jersey.jersey-test-framework:jersey-test-framework-grizzly2:1.9:compile com.sun.xml.bind:jaxb-impl:2.2.3-1:compile com.thoughtworks.paranamer:paranamer:2.6:compile @@ -46,7 +42,6 @@ io.netty:netty:3.8.0.Final:compile io.netty:netty-all:4.0.23.Final:compile javax.activation:activation:1.1:compile javax.inject:javax.inject:1:compile -javax.servlet:javax.servlet-api:3.0.1:compile javax.xml.bind:jaxb-api:2.2.2:compile jline:jline:0.9.94:compile log4j:log4j:1.2.17:compile @@ -88,14 +83,6 @@ org.codehaus.jackson:jackson-mapper-asl:1.8.8:compile org.codehaus.jackson:jackson-xc:1.8.3:compile org.codehaus.jettison:jettison:1.1:compile org.eclipse.jetty.orbit:javax.servlet:3.0.0.v201112011016:compile -org.glassfish:javax.servlet:3.1:compile -org.glassfish.external:management-api:3.0.0-b012:compile -org.glassfish.gmbal:gmbal-api-only:3.0.0-b023:compile -org.glassfish.grizzly:grizzly-framework:2.1.2:compile -org.glassfish.grizzly:grizzly-http:2.1.2:compile -org.glassfish.grizzly:grizzly-http-server:2.1.2:compile -org.glassfish.grizzly:grizzly-http-servlet:2.1.2:compile -org.glassfish.grizzly:grizzly-rcm:2.1.2:compile org.json4s:json4s-ast_2.11:3.2.10:compile org.json4s:json4s-core_2.11:3.2.10:compile org.json4s:json4s-jackson_2.11:3.2.10:compile diff --git a/tests/shared/src/test/scala/coursier/test/ActivationTests.scala b/tests/shared/src/test/scala/coursier/test/ActivationTests.scala new file mode 100644 index 000000000..12a8117a6 --- /dev/null +++ b/tests/shared/src/test/scala/coursier/test/ActivationTests.scala @@ -0,0 +1,325 @@ +package coursier.test + +import coursier.core.{Activation, Parse} +import coursier.core.Activation.Os +import utest._ + +import scalaz.{-\/, \/-} + +object ActivationTests extends TestSuite { + + def parseVersion(s: String) = Parse.version(s).getOrElse(???) + def parseVersionInterval(s: String) = Parse.versionInterval(s).getOrElse(???) + + val macOs = Os( + Some("x86_64"), + Set("mac", "unix"), + Some("mac os x"), + Some("10.12") + ) + + val jdkVersion = parseVersion("1.8.0_112") + + // missing: + // - condition on OS or JDK, but no OS or JDK info provided (-> no match) + // - negated OS infos (starting with "!") - not implemented yet + + val tests = TestSuite { + 'OS - { + 'fromProperties - { + 'MacOSX - { + val props = Map( + "os.arch" -> "x86_64", + "os.name" -> "Mac OS X", + "os.version" -> "10.12", + "path.separator" -> ":" + ) + + val expectedOs = Os( + Some("x86_64"), + Set("mac", "unix"), + Some("mac os x"), + Some("10.12") + ) + + val os = Os.fromProperties(props) + + assert(os == expectedOs) + } + + 'linuxPi - { + val props = Map( + "os.arch" -> "arm", + "os.name" -> "Linux", + "os.version" -> "4.1.13-v7+", + "path.separator" -> ":" + ) + + val expectedOs = Os( + Some("arm"), + Set("unix"), + Some("linux"), + Some("4.1.13-v7+") + ) + + val os = Os.fromProperties(props) + + assert(os == expectedOs) + } + } + + 'active - { + + 'arch - { + val activation = Os(Some("x86_64"), Set(), None, None) + + val isActive = activation.isActive(macOs) + + assert(isActive) + } + + 'wrongArch - { + val activation = Os(Some("arm"), Set(), None, None) + + val isActive = activation.isActive(macOs) + + assert(!isActive) + } + + 'family - { + val activation = Os(None, Set("mac"), None, None) + + val isActive = activation.isActive(macOs) + + assert(isActive) + } + + 'wrongFamily - { + val activation = Os(None, Set("windows"), None, None) + + val isActive = activation.isActive(macOs) + + assert(!isActive) + } + + 'name - { + val activation = Os(None, Set(), Some("mac os x"), None) + + val isActive = activation.isActive(macOs) + + assert(isActive) + } + + 'wrongName - { + val activation = Os(None, Set(), Some("linux"), None) + + val isActive = activation.isActive(macOs) + + assert(!isActive) + } + + 'version - { + val activation = Os(None, Set(), None, Some("10.12")) + + val isActive = activation.isActive(macOs) + + assert(isActive) + } + + 'wrongVersion - { + val activation = Os(None, Set(), None, Some("10.11")) + + val isActive = activation.isActive(macOs) + + assert(!isActive) + } + } + } + + 'properties - { + val activation = Activation.empty.copy( + properties = Seq( + "required" -> None, + "requiredWithValue" -> Some("foo"), + "requiredWithNegValue" -> Some("!bar") + ) + ) + + 'match - { + val isActive = activation.isActive( + Map( + "required" -> "a", + "requiredWithValue" -> "foo", + "requiredWithNegValue" -> "baz" + ), + Os.empty, + None + ) + + assert(isActive) + } + + 'noMatch - { + * - { + val isActive = activation.isActive( + Map( + "requiredWithValue" -> "foo", + "requiredWithNegValue" -> "baz" + ), + Os.empty, + None + ) + + assert(!isActive) + } + + * - { + val isActive = activation.isActive( + Map( + "required" -> "a", + "requiredWithValue" -> "fooz", + "requiredWithNegValue" -> "baz" + ), + Os.empty, + None + ) + + assert(!isActive) + } + + * - { + val isActive = activation.isActive( + Map( + "required" -> "a", + "requiredWithValue" -> "foo", + "requiredWithNegValue" -> "bar" + ), + Os.empty, + None + ) + + assert(!isActive) + } + } + } + + 'jdkVersion - { + + 'match - { + 'exactVersion - { + val activation = Activation( + Nil, + Os.empty, + Some(\/-(Seq(parseVersion("1.8.0_112")))) + ) + + val isActive = activation.isActive(Map(), Os.empty, Some(jdkVersion)) + + assert(isActive) + } + + 'exactVersionSeveral - { + val activation = Activation( + Nil, + Os.empty, + Some(\/-(Seq(parseVersion("1.8.0_102"), parseVersion("1.8.0_112")))) + ) + + val isActive = activation.isActive(Map(), Os.empty, Some(jdkVersion)) + + assert(isActive) + } + + + 'wrongExactVersion - { + val activation = Activation( + Nil, + Os.empty, + Some(\/-(Seq(parseVersion("1.8.0_102")))) + ) + + val isActive = activation.isActive(Map(), Os.empty, Some(jdkVersion)) + + assert(!isActive) + } + + + 'wrongExactVersionSeveral - { + val activation = Activation( + Nil, + Os.empty, + Some(\/-(Seq(parseVersion("1.8.0_92"), parseVersion("1.8.0_102")))) + ) + + val isActive = activation.isActive(Map(), Os.empty, Some(jdkVersion)) + + assert(!isActive) + } + + 'versionInterval - { + val activation = Activation( + Nil, + Os.empty, + Some(-\/(parseVersionInterval("[1.8,)"))) + ) + + val isActive = activation.isActive(Map(), Os.empty, Some(jdkVersion)) + + assert(isActive) + } + + 'wrongVersionInterval - { + val activation = Activation( + Nil, + Os.empty, + Some(-\/(parseVersionInterval("[1.7,1.8)"))) + ) + + val isActive = activation.isActive(Map(), Os.empty, Some(jdkVersion)) + + assert(!isActive) + } + } + } + + 'all - { + val activation = Activation( + Seq( + "required" -> None, + "requiredWithValue" -> Some("foo"), + "requiredWithNegValue" -> Some("!bar") + ), + Os(None, Set("mac"), None, None), + Some(-\/(parseVersionInterval("[1.8,)"))) + ) + + 'match - { + val isActive = activation.isActive( + Map( + "required" -> "a", + "requiredWithValue" -> "foo", + "requiredWithNegValue" -> "baz" + ), + macOs, + Some(jdkVersion) + ) + + assert(isActive) + } + + 'noMatch - { + val isActive = activation.isActive( + Map( + "requiredWithValue" -> "foo", + "requiredWithNegValue" -> "baz" + ), + macOs, + Some(jdkVersion) + ) + + assert(!isActive) + } + } + } + +} diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index b6a3a27bd..addecc379 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -19,16 +19,14 @@ object CentralTests extends TestSuite { deps: Set[Dependency], filter: Option[Dependency => Boolean] = None, extraRepo: Option[Repository] = None, - profiles: Set[String] = Set.empty + profiles: Option[Set[String]] = None ) = { val repositories0 = extraRepo.toSeq ++ repositories Resolution( deps, filter = filter, - profileActivation = Some( - core.Resolution.userProfileActivation(profiles) - ) + userActivations = profiles.map(_.iterator.map(_ -> true).toMap) ) .process .run(repositories0) @@ -40,7 +38,7 @@ object CentralTests extends TestSuite { version: String, extraRepo: Option[Repository] = None, configuration: String = "", - profiles: Set[String] = Set.empty + profiles: Option[Set[String]] = None ) = async { val attrPathPart = @@ -157,7 +155,7 @@ object CentralTests extends TestSuite { 'logback - { async { val dep = Dependency(Module("ch.qos.logback", "logback-classic"), "1.1.3") - val res = await(resolve(Set(dep))).clearCaches.clearProfileActivation + val res = await(resolve(Set(dep))).clearCaches val expected = Resolution( rootDependencies = Set(dep), @@ -173,7 +171,7 @@ object CentralTests extends TestSuite { 'asm - { async { val dep = Dependency(Module("org.ow2.asm", "asm-commons"), "5.0.2") - val res = await(resolve(Set(dep))).clearCaches.clearProfileActivation + val res = await(resolve(Set(dep))).clearCaches val expected = Resolution( rootDependencies = Set(dep), @@ -190,7 +188,7 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("joda-time", "joda-time"), "[2.2,2.8]") val res0 = await(resolve(Set(dep))) - val res = res0.clearCaches.clearProfileActivation + val res = res0.clearCaches val expected = Resolution( rootDependencies = Set(dep), @@ -209,7 +207,7 @@ object CentralTests extends TestSuite { resolutionCheck( Module("org.apache.spark", "spark-core_2.11"), "1.3.1", - profiles = Set("hadoop-2.2") + profiles = Some(Set("hadoop-2.2")) ) } diff --git a/tests/shared/src/test/scala/coursier/test/package.scala b/tests/shared/src/test/scala/coursier/test/package.scala index 7a2266b6d..2ed9fbb30 100644 --- a/tests/shared/src/test/scala/coursier/test/package.scala +++ b/tests/shared/src/test/scala/coursier/test/package.scala @@ -23,15 +23,13 @@ package object test { ) def clearFilter: Resolution = underlying.copy(filter = None) - def clearProfileActivation: Resolution = - underlying.copy(profileActivation = None) } object Profile { type Activation = core.Activation object Activation { def apply(properties: Seq[(String, Option[String])] = Nil): Activation = - core.Activation(properties) + core.Activation(properties, coursier.core.Activation.Os.empty, None) } def apply( From f1e1b6aac92c9bd2b99351692b383a681dd6b0b1 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Mon, 7 Nov 2016 11:22:52 +0100 Subject: [PATCH 2/4] Limit messages more in SBT plugin Only print "Updating ...", "Fetching artifacts of ..." messages if something is actually downloaded --- .../src/main/scala/coursier/TermDisplay.scala | 32 +- .../src/main/scala-2.10/coursier/Tasks.scala | 285 +++++++++--------- 2 files changed, 176 insertions(+), 141 deletions(-) diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index 7823a3a9d..202a5848c 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -162,6 +162,7 @@ object TermDisplay { } private class UpdateDisplayRunnable( + beforeOutput: => Unit, out: Writer, width: Int, fallbackMode: Boolean @@ -170,6 +171,9 @@ object TermDisplay { import Terminal.Ansi private var currentHeight = 0 + private var printedAnything0 = false + + def printedAnything() = printedAnything0 private val needsUpdate = new AtomicBoolean(false) @@ -305,6 +309,11 @@ object TermDisplay { for ((url, info) <- done0 ++ downloads0) { assert(info != null, s"Incoherent state ($url)") + if (!printedAnything0) { + beforeOutput + printedAnything0 = true + } + truncatedPrintln(url) out.clearLine(2) out.write(s" ${info.display()}\n") @@ -422,19 +431,36 @@ class TermDisplay( else 1000L / 60 - def init(): Unit = { - updateRunnableOpt = Some(new UpdateDisplayRunnable(out, width, fallbackMode0)) + /*** + * + * @param beforeOutput: called before any output is printed, iff something else is outputed. + * (That is, if that `TermDisplay` doesn't print any progress, + * `initialMessage` won't be printed either.) + */ + def init(beforeOutput: => Unit): Unit = { + updateRunnableOpt = Some(new UpdateDisplayRunnable(beforeOutput, out, width, fallbackMode0)) updateRunnable.init() scheduler.scheduleAtFixedRate(updateRunnable, 0L, refreshInterval, TimeUnit.MILLISECONDS) } - def stop(): Unit = { + def init(): Unit = + init(()) + + /** + * + * @return: whether any message was printed by this `TermDisplay` + */ + def stopDidPrintSomething(): Boolean = { scheduler.shutdown() scheduler.awaitTermination(refreshInterval, TimeUnit.MILLISECONDS) updateRunnable.cleanDisplay() + updateRunnable.printedAnything() } + def stop(): Unit = + stopDidPrintSomething() + override def downloadingArtifact(url: String, file: File): Unit = updateRunnable.newEntry( url, diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index c1d773d81..4e2cedc5d 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -579,7 +579,7 @@ object Tasks { var pool: ExecutorService = null var resLogger: TermDisplay = null - try { + val res = try { pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) resLogger = createLogger() @@ -615,17 +615,26 @@ object Tasks { ) } - if (verbosityLevel >= 0) - log.info( - s"Updating $projectName" + (if (sbtClassifiers) " (sbt classifiers)" else "") - ) - if (verbosityLevel >= 2) - for (depRepr <- depsRepr(currentProject.dependencies)) - log.info(s" $depRepr") + val initialMessage = + Seq( + if (verbosityLevel >= 0) + Seq(s"Updating $projectName" + (if (sbtClassifiers) " (sbt classifiers)" else "")) + else + Nil, + if (verbosityLevel >= 2) + depsRepr(currentProject.dependencies).map(depRepr => + s" $depRepr" + ) + else + Nil + ).flatten.mkString("\n") - resLogger.init() + if (verbosityLevel >= 1) + log.info(initialMessage) - val res = startRes + resLogger.init(if (verbosityLevel < 1) log.info(initialMessage)) + + startRes .process .run(fetch, maxIterations) .attemptRun @@ -634,46 +643,44 @@ object Tasks { .throwException() ) .merge - - if (!res.isDone) - ResolutionError.MaximumIterationsReached - .throwException() - - if (res.conflicts.nonEmpty) { - val projCache = res.projectCache.mapValues { case (_, p) => p } - - ResolutionError.Conflicts( - "Conflict(s) in dependency resolution:\n " + - Print.dependenciesUnknownConfigs(res.conflicts.toVector, projCache) - ).throwException() - } - - if (res.errors.nonEmpty) { - val internalRepositoriesLen = internalRepositories.length - val errors = - if (repositories.length > internalRepositoriesLen) - // drop internal repository errors - res.errors.map { - case (dep, errs) => - dep -> errs.drop(internalRepositoriesLen) - } - else - res.errors - - ResolutionError.MetadataDownloadErrors(errors) - .throwException() - } - - if (verbosityLevel >= 0) - log.info(s"Resolved $projectName dependencies") - - res } finally { if (pool != null) pool.shutdown() if (resLogger != null) - resLogger.stop() + if ((resLogger.stopDidPrintSomething() && verbosityLevel >= 0) || verbosityLevel >= 1) + log.info(s"Resolved $projectName dependencies") } + + if (!res.isDone) + ResolutionError.MaximumIterationsReached + .throwException() + + if (res.conflicts.nonEmpty) { + val projCache = res.projectCache.mapValues { case (_, p) => p } + + ResolutionError.Conflicts( + "Conflict(s) in dependency resolution:\n " + + Print.dependenciesUnknownConfigs(res.conflicts.toVector, projCache) + ).throwException() + } + + if (res.errors.nonEmpty) { + val internalRepositoriesLen = internalRepositories.length + val errors = + if (repositories.length > internalRepositoriesLen) + // drop internal repository errors + res.errors.map { + case (dep, errs) => + dep -> errs.drop(internalRepositoriesLen) + } + else + res.errors + + ResolutionError.MetadataDownloadErrors(errors) + .throwException() + } + + res } resolutionsCache.getOrElseUpdate( @@ -777,45 +784,45 @@ object Tasks { }.value def report = { + + val depsByConfig = grouped(currentProject.dependencies) + + val configs = coursierConfigurations.value + + if (verbosityLevel >= 2) { + val finalDeps = Config.dependenciesWithConfig( + res, + depsByConfig.map { case (k, l) => k -> l.toSet }, + configs + ) + + val projCache = res.projectCache.mapValues { case (_, p) => p } + val repr = Print.dependenciesUnknownConfigs(finalDeps.toVector, projCache) + log.info(repr.split('\n').map(" " + _).mkString("\n")) + } + + val classifiers = + if (withClassifiers) + Some { + if (sbtClassifiers) + cm.classifiers + else + transitiveClassifiers.value + } + else + None + + val allArtifacts = + classifiers match { + case None => res.artifacts + case Some(cl) => res.classifiersArtifacts(cl) + } + var pool: ExecutorService = null var artifactsLogger: TermDisplay = null - try { + val artifactFilesOrErrors = try { pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) - - val depsByConfig = grouped(currentProject.dependencies) - - val configs = coursierConfigurations.value - - if (verbosityLevel >= 2) { - val finalDeps = Config.dependenciesWithConfig( - res, - depsByConfig.map { case (k, l) => k -> l.toSet }, - configs - ) - - val projCache = res.projectCache.mapValues { case (_, p) => p } - val repr = Print.dependenciesUnknownConfigs(finalDeps.toVector, projCache) - log.info(repr.split('\n').map(" " + _).mkString("\n")) - } - - val classifiers = - if (withClassifiers) - Some { - if (sbtClassifiers) - cm.classifiers - else - transitiveClassifiers.value - } - else - None - - val allArtifacts = - classifiers match { - case None => res.artifacts - case Some(cl) => res.classifiersArtifacts(cl) - } - artifactsLogger = createLogger() val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => @@ -836,79 +843,81 @@ object Tasks { .map((a, _)) } - if (verbosityLevel >= 0) - log.info( + val artifactInitialMessage = + if (verbosityLevel >= 0) s"Fetching artifacts of $projectName" + (if (sbtClassifiers) " (sbt classifiers)" else "") - ) + else + "" - artifactsLogger.init() + if (verbosityLevel >= 1) + log.info(artifactInitialMessage) - val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { + artifactsLogger.init(if (verbosityLevel < 1) log.info(artifactInitialMessage)) + + Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { case -\/(ex) => ResolutionError.UnknownDownloadException(ex) .throwException() case \/-(l) => l.toMap } - - if (verbosityLevel >= 0) - log.info( - s"Fetched artifacts of $projectName" + - (if (sbtClassifiers) " (sbt classifiers)" else "") - ) - - val artifactFiles = artifactFilesOrErrors.collect { - case (artifact, \/-(file)) => - artifact -> file - } - - val artifactErrors = artifactFilesOrErrors.toVector.collect { - case (_, -\/(err)) => - err - } - - if (artifactErrors.nonEmpty) { - val error = ResolutionError.DownloadErrors(artifactErrors) - - if (ignoreArtifactErrors) - log.warn(error.description(verbosityLevel >= 1)) - else - error.throwException() - } - - // can be non empty only if ignoreArtifactErrors is true - val erroredArtifacts = artifactFilesOrErrors.collect { - case (artifact, -\/(_)) => - artifact - }.toSet - - def artifactFileOpt(artifact: Artifact) = { - val artifact0 = artifact - .copy(attributes = Attributes()) // temporary hack :-( - val res = artifactFiles.get(artifact0) - - if (res.isEmpty && !erroredArtifacts(artifact0)) - log.error(s"${artifact.url} not downloaded (should not happen)") - - res - } - - writeIvyFiles() - - ToSbt.updateReport( - depsByConfig, - res, - configs, - classifiers, - artifactFileOpt - ) } finally { if (pool != null) pool.shutdown() if (artifactsLogger != null) - artifactsLogger.stop() + if ((artifactsLogger.stopDidPrintSomething() && verbosityLevel >= 0) || verbosityLevel >= 1) + log.info( + s"Fetched artifacts of $projectName" + + (if (sbtClassifiers) " (sbt classifiers)" else "") + ) } + + val artifactFiles = artifactFilesOrErrors.collect { + case (artifact, \/-(file)) => + artifact -> file + } + + val artifactErrors = artifactFilesOrErrors.toVector.collect { + case (_, -\/(err)) => + err + } + + if (artifactErrors.nonEmpty) { + val error = ResolutionError.DownloadErrors(artifactErrors) + + if (ignoreArtifactErrors) + log.warn(error.description(verbosityLevel >= 1)) + else + error.throwException() + } + + // can be non empty only if ignoreArtifactErrors is true + val erroredArtifacts = artifactFilesOrErrors.collect { + case (artifact, -\/(_)) => + artifact + }.toSet + + def artifactFileOpt(artifact: Artifact) = { + val artifact0 = artifact + .copy(attributes = Attributes()) // temporary hack :-( + val res = artifactFiles.get(artifact0) + + if (res.isEmpty && !erroredArtifacts(artifact0)) + log.error(s"${artifact.url} not downloaded (should not happen)") + + res + } + + writeIvyFiles() + + ToSbt.updateReport( + depsByConfig, + res, + configs, + classifiers, + artifactFileOpt + ) } reportsCache.getOrElseUpdate( From 73243f54e304d7a7b308c8050b926526fa244fcd Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Mon, 7 Nov 2016 11:22:55 +0100 Subject: [PATCH 3/4] Fix minor TermDisplay glitch Display cleaning possibly made too early --- cache/src/main/scala/coursier/TermDisplay.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index 202a5848c..89b8b1db4 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -453,7 +453,7 @@ class TermDisplay( */ def stopDidPrintSomething(): Boolean = { scheduler.shutdown() - scheduler.awaitTermination(refreshInterval, TimeUnit.MILLISECONDS) + scheduler.awaitTermination(2 * refreshInterval, TimeUnit.MILLISECONDS) updateRunnable.cleanDisplay() updateRunnable.printedAnything() } From d5cc47ae5f0798b5354d53086d7662ed86fa63a4 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Mon, 7 Nov 2016 11:22:57 +0100 Subject: [PATCH 4/4] Add non regression test for org.nd4j:nd4j-native:0.5.0... ...and corresponding fixes / workarounds --- .../main/scala/coursier/core/Activation.scala | 8 +++++++- .../main/scala/coursier/core/Resolution.scala | 10 +++++++++- .../resolutions/org.nd4j/nd4j-native/0.5.0 | 16 ++++++++++++++++ .../test/scala/coursier/test/CentralTests.scala | 11 +++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 tests/shared/src/test/resources/resolutions/org.nd4j/nd4j-native/0.5.0 diff --git a/core/shared/src/main/scala/coursier/core/Activation.scala b/core/shared/src/main/scala/coursier/core/Activation.scala index bf10b82d5..79e980815 100644 --- a/core/shared/src/main/scala/coursier/core/Activation.scala +++ b/core/shared/src/main/scala/coursier/core/Activation.scala @@ -56,8 +56,14 @@ object Activation { def isEmpty: Boolean = arch.isEmpty && families.isEmpty && name.isEmpty && version.isEmpty + def archMatch(current: Option[String]): Boolean = + arch.forall(current.toSeq.contains) || { + // seems required by org.nd4j:nd4j-native:0.5.0 + arch.toSeq.contains("x86-64") && current.toSeq.contains("x86_64") + } + def isActive(osInfo: Os): Boolean = - arch.forall(osInfo.arch.toSeq.contains) && + archMatch(osInfo.arch) && families.forall { f => if (Os.knownFamilies(f)) osInfo.families.contains(f) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index d7b6e6438..b97055a83 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -363,6 +363,12 @@ object Resolution { def projectProperties(project: Project): Seq[(String, String)] = { + // vague attempt at recovering the POM packaging tag + def packagingOpt = project.publications.collectFirst { + case ("compile", pub) => + pub.`type` + } + // FIXME The extra properties should only be added for Maven projects, not Ivy ones val properties0 = Seq( // some artifacts seem to require these (e.g. org.jmock:jmock-legacy:2.5.1) @@ -374,7 +380,9 @@ object Resolution { "project.groupId" -> project.module.organization, "project.artifactId" -> project.module.name, "project.version" -> project.version - ) ++ project.parent.toSeq.flatMap { + ) ++ packagingOpt.toSeq.map { packaging => + "project.packaging" -> packaging + } ++ project.parent.toSeq.flatMap { case (parModule, parVersion) => Seq( "project.parent.groupId" -> parModule.organization, diff --git a/tests/shared/src/test/resources/resolutions/org.nd4j/nd4j-native/0.5.0 b/tests/shared/src/test/resources/resolutions/org.nd4j/nd4j-native/0.5.0 new file mode 100644 index 000000000..83dc7191a --- /dev/null +++ b/tests/shared/src/test/resources/resolutions/org.nd4j/nd4j-native/0.5.0 @@ -0,0 +1,16 @@ +com.google.code.findbugs:annotations:2.0.1:compile +com.google.guava:guava:18.0:compile +commons-io:commons-io:2.4:compile +org.apache.commons:commons-lang3:3.3.1:compile +org.apache.commons:commons-math3:3.4.1:compile +org.bytedeco:javacpp:1.2.3:compile +org.javassist:javassist:3.18.2-GA:compile +org.nd4j:nd4j-api:0.5.0:compile +org.nd4j:nd4j-buffer:0.5.0:compile +org.nd4j:nd4j-common:0.5.0:compile +org.nd4j:nd4j-context:0.5.0:compile +org.nd4j:nd4j-native:0.5.0:compile +org.nd4j:nd4j-native-api:0.5.0:compile +org.projectlombok:lombok:1.16.4:compile +org.reflections:reflections:0.9.10:compile +org.slf4j:slf4j-api:1.7.10:compile diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index addecc379..3c6030e7e 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -436,6 +436,17 @@ object CentralTests extends TestSuite { "1.11-8" ) } + + 'nd4jNative - { + // In particular: + // - uses OS-based activation, + // - requires converting a "x86-64" to "x86_64" in it, and + // - uses "project.packaging" property + resolutionCheck( + Module("org.nd4j", "nd4j-native"), + "0.5.0" + ) + } } }