From 68768cc87154109f2031c1374f1f2f70df932533 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 10 Jan 2016 21:32:28 +0100 Subject: [PATCH 1/5] Add resolve command, plus some tidying --- .../src/main/scala/coursier/CacheParse.scala | 54 ++++ .../scala-2.11/coursier/cli/Coursier.scala | 104 +++---- .../main/scala-2.11/coursier/cli/Helper.scala | 254 ++++++------------ .../src/main/scala/coursier/util/Config.scala | 49 ++++ .../src/main/scala/coursier/util/Parse.scala | 70 +++++ .../src/main/scala/coursier/util/Print.scala | 30 +++ 6 files changed, 347 insertions(+), 214 deletions(-) create mode 100644 cache/src/main/scala/coursier/CacheParse.scala create mode 100644 core/shared/src/main/scala/coursier/util/Config.scala create mode 100644 core/shared/src/main/scala/coursier/util/Parse.scala create mode 100644 core/shared/src/main/scala/coursier/util/Print.scala diff --git a/cache/src/main/scala/coursier/CacheParse.scala b/cache/src/main/scala/coursier/CacheParse.scala new file mode 100644 index 000000000..69409ecad --- /dev/null +++ b/cache/src/main/scala/coursier/CacheParse.scala @@ -0,0 +1,54 @@ +package coursier + +import coursier.ivy.IvyRepository +import coursier.util.Parse + +import scalaz._, Scalaz._ + +object CacheParse { + + def repository(s: String): Validation[String, Repository] = + if (s == "ivy2local" || s == "ivy2Local") + Cache.ivy2Local.success + else { + val repo = Parse.repository(s) + + val url = repo match { + case m: MavenRepository => + m.root + case i: IvyRepository => + i.pattern + case r => + sys.error(s"Unrecognized repository: $r") + } + + if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/")) + repo.success + else + s"Unrecognized protocol in $url".failure + } + + def repositories(l: Seq[String]): ValidationNel[String, Seq[Repository]] = + l.toVector.traverseU { s => + repository(s).leftMap(_.wrapNel) + } + + def cachePolicies(s: String): ValidationNel[String, Seq[CachePolicy]] = + s.split(',').toVector.traverseU { + case "offline" => + Seq(CachePolicy.LocalOnly).successNel + case "update-changing" => + Seq(CachePolicy.UpdateChanging).successNel + case "update" => + Seq(CachePolicy.Update).successNel + case "missing" => + Seq(CachePolicy.FetchMissing).successNel + case "force" => + Seq(CachePolicy.ForceDownload).successNel + case "default" => + Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing).successNel + case other => + s"Unrecognized mode: $other".failureNel + }.map(_.flatten) + +} diff --git a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala index a79c57f11..6c91775dd 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala @@ -1,45 +1,45 @@ package coursier package cli -import java.io.{ByteArrayOutputStream, FileOutputStream, File, IOException} +import java.io.{ ByteArrayOutputStream, File, IOException } import java.net.URLClassLoader import java.nio.file.{ Files => NIOFiles } -import java.nio.file.attribute.{FileTime, PosixFilePermission} +import java.nio.file.attribute.PosixFilePermission import java.util.Properties -import java.util.zip.{ZipEntry, ZipOutputStream, ZipInputStream, ZipFile} +import java.util.zip.{ ZipEntry, ZipOutputStream, ZipInputStream } -import caseapp._ +import caseapp.{ HelpMessage => Help, ValueDescription => Value, ExtraName => Short, _ } import coursier.util.ClasspathFilter case class CommonOptions( - @HelpMessage("Keep optional dependencies (Maven)") + @Help("Keep optional dependencies (Maven)") keepOptional: Boolean, - @HelpMessage("Download mode (default: missing, that is fetch things missing from cache)") - @ValueDescription("offline|update-changing|update|missing|force") - @ExtraName("m") - mode: String = "missing", - @HelpMessage("Quiet output") - @ExtraName("q") + @Help("Download mode (default: missing, that is fetch things missing from cache)") + @Value("offline|update-changing|update|missing|force") + @Short("m") + mode: String = "default", + @Help("Quiet output") + @Short("q") quiet: Boolean, - @HelpMessage("Increase verbosity (specify several times to increase more)") - @ExtraName("v") + @Help("Increase verbosity (specify several times to increase more)") + @Short("v") verbose: List[Unit], - @HelpMessage("Maximum number of resolution iterations (specify a negative value for unlimited, default: 100)") - @ExtraName("N") + @Help("Maximum number of resolution iterations (specify a negative value for unlimited, default: 100)") + @Short("N") maxIterations: Int = 100, - @HelpMessage("Repositories - for multiple repositories, separate with comma and/or repeat this option (e.g. -r central,ivy2local -r sonatype-snapshots, or equivalently -r central,ivy2local,sonatype-snapshots)") - @ExtraName("r") + @Help("Repositories - for multiple repositories, separate with comma and/or repeat this option (e.g. -r central,ivy2local -r sonatype-snapshots, or equivalently -r central,ivy2local,sonatype-snapshots)") + @Short("r") repository: List[String], - @HelpMessage("Do not add default repositories (~/.ivy2/local, and Central)") + @Help("Do not add default repositories (~/.ivy2/local, and Central)") noDefault: Boolean = false, - @HelpMessage("Modify names in Maven repository paths for SBT plugins") + @Help("Modify names in Maven repository paths for SBT plugins") sbtPluginHack: Boolean = false, - @HelpMessage("Force module version") - @ValueDescription("organization:name:forcedVersion") - @ExtraName("V") + @Help("Force module version") + @Value("organization:name:forcedVersion") + @Short("V") forceVersion: List[String], - @HelpMessage("Maximum number of parallel downloads (default: 6)") - @ExtraName("n") + @Help("Maximum number of parallel downloads (default: 6)") + @Short("n") parallel: Int = 6, @Recurse cacheOptions: CacheOptions @@ -48,22 +48,32 @@ case class CommonOptions( } case class CacheOptions( - @HelpMessage("Cache directory (defaults to environment variable COURSIER_CACHE or ~/.coursier/cache/v1)") - @ExtraName("C") + @Help("Cache directory (defaults to environment variable COURSIER_CACHE or ~/.coursier/cache/v1)") + @Short("C") cache: String = Cache.defaultBase.toString ) sealed trait CoursierCommand extends Command +case class Resolve( + @Recurse + common: CommonOptions +) extends CoursierCommand { + + // the `val helper = ` part is needed because of DelayedInit it seems + val helper = new Helper(common, remainingArgs) + +} + case class Fetch( - @HelpMessage("Fetch source artifacts") - @ExtraName("S") + @Help("Fetch source artifacts") + @Short("S") sources: Boolean, - @HelpMessage("Fetch javadoc artifacts") - @ExtraName("D") + @Help("Fetch javadoc artifacts") + @Short("D") javadoc: Boolean, - @HelpMessage("Print java -cp compatible output") - @ExtraName("p") + @Help("Print java -cp compatible output") + @Short("p") classpath: Boolean, @Recurse common: CommonOptions @@ -88,11 +98,11 @@ case class Fetch( } case class Launch( - @ExtraName("M") - @ExtraName("main") + @Short("M") + @Short("main") mainClass: String, - @ExtraName("c") - @HelpMessage("Assume coursier is a dependency of the launched app, and share the coursier dependency of the launcher with it - allows the launched app to get the resolution that launched it via ResolutionClassLoader") + @Short("c") + @Help("Assume coursier is a dependency of the launched app, and share the coursier dependency of the launcher with it - allows the launched app to get the resolution that launched it via ResolutionClassLoader") addCoursier: Boolean, @Recurse common: CommonOptions @@ -200,21 +210,21 @@ case class Launch( } case class Bootstrap( - @ExtraName("M") - @ExtraName("main") + @Short("M") + @Short("main") mainClass: String, - @ExtraName("o") + @Short("o") output: String = "bootstrap", - @ExtraName("D") + @Short("D") downloadDir: String, - @ExtraName("f") + @Short("f") force: Boolean, - @HelpMessage(s"Internal use - prepend base classpath options to arguments (like -B jar1 -B jar2 etc.)") - @ExtraName("b") + @Help(s"Internal use - prepend base classpath options to arguments (like -B jar1 -B jar2 etc.)") + @Short("b") prependClasspath: Boolean, - @HelpMessage("Set environment variables in the generated launcher. No escaping is done. Value is simply put between quotes in the launcher preamble.") - @ValueDescription("key=value") - @ExtraName("P") + @Help("Set environment variables in the generated launcher. No escaping is done. Value is simply put between quotes in the launcher preamble.") + @Value("key=value") + @Short("P") property: List[String], @Recurse common: CommonOptions @@ -365,7 +375,7 @@ case class Bootstrap( case class BaseCommand( @Hidden - @ExtraName("B") + @Short("B") baseCp: List[String] ) extends Command { Coursier.baseCp = baseCp diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index b4fb43fe2..3959274b6 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -5,8 +5,9 @@ import java.io.{ OutputStreamWriter, File } import java.util.concurrent.Executors import coursier.ivy.IvyRepository +import coursier.util.{Print, Parse} -import scalaz.{ \/-, -\/ } +import scalaz.{Failure, Success, \/-, -\/} import scalaz.concurrent.{ Task, Strategy } object Helper { @@ -33,6 +34,28 @@ object Helper { } } +object Util { + + def prematureExit(msg: String): Nothing = { + Console.err.println(msg) + sys.exit(255) + } + + def prematureExitIf(cond: Boolean)(msg: => String): Unit = + if (cond) + prematureExit(msg) + + def exit(msg: String): Nothing = { + Console.err.println(msg) + sys.exit(1) + } + + def exitIf(cond: Boolean)(msg: => String): Unit = + if (cond) + exit(msg) + +} + class Helper( common: CommonOptions, remainingArgs: Seq[String] @@ -40,22 +63,16 @@ class Helper( import common._ import Helper.errPrintln - val cachePolicies = mode match { - case "offline" => - Seq(CachePolicy.LocalOnly) - case "update-changing" => - Seq(CachePolicy.UpdateChanging) - case "update" => - Seq(CachePolicy.Update) - case "missing" => - Seq(CachePolicy.FetchMissing) - case "force" => - Seq(CachePolicy.ForceDownload) - case "default" => - Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing) - case other => - errPrintln(s"Unrecognized mode: $other") - sys.exit(255) + import Util._ + + val cachePoliciesValidation = CacheParse.cachePolicies(common.mode) + + val cachePolicies = cachePoliciesValidation match { + case Success(cp) => cp + case Failure(errors) => + prematureExit( + s"Error parsing modes:\n${errors.list.map(" "+_).mkString("\n")}" + ) } val caches = @@ -66,58 +83,30 @@ class Helper( val pool = Executors.newFixedThreadPool(parallel, Strategy.DefaultDaemonThreadFactory) - val central = MavenRepository("https://repo1.maven.org/maven2/") - val defaultRepositories = Seq( Cache.ivy2Local, - central + MavenRepository("https://repo1.maven.org/maven2") ) - val repositories0 = common.repository.map { repo => - val repo0 = repo.toLowerCase - if (repo0 == "central") - Right(central) - else if (repo0 == "ivy2local") - Right(Cache.ivy2Local) - else if (repo0.startsWith("sonatype:")) - Right( - MavenRepository(s"https://oss.sonatype.org/content/repositories/${repo.drop("sonatype:".length)}") - ) - else { - val (url, r) = - if (repo.startsWith("ivy:")) { - val url = repo.drop("ivy:".length) - (url, IvyRepository(url)) - } else - (repo, MavenRepository(repo)) - - if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/")) - Right(r) - else - Left(repo -> s"Unrecognized protocol or repository: $url") - } - } - - val unrecognizedRepos = repositories0.collect { case Left(e) => e } - if (unrecognizedRepos.nonEmpty) { - errPrintln(s"${unrecognizedRepos.length} error(s) parsing repositories:") - for ((repo, err) <- unrecognizedRepos) - errPrintln(s"$repo: $err") - sys.exit(255) - } - - val repositories1 = - (if (common.noDefault) Nil else defaultRepositories) ++ - repositories0.collect { case Right(r) => r } - - val repositories = + val repositoriesValidation = CacheParse.repositories(common.repository).map { repos0 => + val repos = (if (common.noDefault) Nil else defaultRepositories) ++ repos0 if (common.sbtPluginHack) - repositories1.map { + repos.map { case m: MavenRepository => m.copy(sbtAttrStub = true) case other => other } else - repositories1 + repos + } + + val repositories = repositoriesValidation match { + case Success(repos) => repos + case Failure(errors) => + prematureExit( + s"Error parsing repositories:\n${errors.list.map(" "+_).mkString("\n")}" + ) + } + val (rawDependencies, extraArgs) = { val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) @@ -128,63 +117,25 @@ class Helper( } } - val (splitDependencies, malformed) = rawDependencies.toList - .map(_.split(":", 3).toSeq) - .partition(_.length == 3) + val (modVerErrors, moduleVersions) = Parse.moduleVersions(remainingArgs) - val (splitForceVersions, malformedForceVersions) = forceVersion - .map(_.split(":", 3).toSeq) - .partition(_.length == 3) - - if (splitDependencies.isEmpty) { - Console.err.println(s"Error: no dependencies specified.") - // CaseApp.printUsage[Coursier]() - sys exit 1 + prematureExitIf(modVerErrors.nonEmpty) { + s"Cannot parse dependencies:\n" + modVerErrors.map(" "+_).mkString("\n") } - if (malformed.nonEmpty || malformedForceVersions.nonEmpty) { - if (malformed.nonEmpty) { - errPrintln("Malformed dependency(ies), should be like org:name:version") - for (s <- malformed) - errPrintln(s" ${s.mkString(":")}") - } - - if (malformedForceVersions.nonEmpty) { - errPrintln("Malformed force version(s), should be like org:name:forcedVersion") - for (s <- malformedForceVersions) - errPrintln(s" ${s.mkString(":")}") - } - - sys.exit(1) + val dependencies = moduleVersions.map { + case (module, version) => + Dependency(module, version, configuration = "default(compile)") } - val moduleVersions = splitDependencies.map{ - case Seq(org, namePart, version) => - val p = namePart.split(';') - val name = p.head - val splitAttributes = p.tail.map(_.split("=", 2).toSeq).toSeq - val malformedAttributes = splitAttributes.filter(_.length != 2) - if (malformedAttributes.nonEmpty) { - // FIXME Get these for all dependencies at once - Console.err.println(s"Malformed attributes in ${splitDependencies.mkString(":")}") - // :( - sys.exit(255) - } - val attributes = splitAttributes.collect { - case Seq(k, v) => k -> v - } - (Module(org, name, attributes.toMap), version) - } - val deps = moduleVersions.map{case (mod, ver) => - Dependency(mod, ver, configuration = "runtime") + val (forceVersionErrors, forceVersions0) = Parse.moduleVersions(forceVersion) + + prematureExitIf(forceVersionErrors.nonEmpty) { + s"Cannot parse forced versions:\n" + forceVersionErrors.map(" "+_).mkString("\n") } val forceVersions = { - val forceVersions0 = splitForceVersions.map { - case Seq(org, name, version) => (Module(org, name), version) - } - val grouped = forceVersions0 .groupBy { case (mod, _) => mod } .map { case (mod, l) => mod -> l.map { case (_, version) => version } } @@ -196,7 +147,7 @@ class Helper( } val startRes = Resolution( - deps.toSet, + dependencies.toSet, forceVersions = forceVersions, filter = Some(dep => keepOptional || !dep.optional) ) @@ -219,17 +170,21 @@ class Helper( if (verbose0 <= 0) fetchQuiet else { modVers: Seq[(Module, String)] => - val print = Task{ + val print = Task { errPrintln(s"Getting ${modVers.length} project definition(s)") } print.flatMap(_ => fetchQuiet(modVers)) } + def indent(s: String): String = + if (s.isEmpty) + s + else + s.split('\n').map(" "+_).mkString("\n") + if (verbose0 >= 0) { - errPrintln("Dependencies:") - for ((mod, ver) <- moduleVersions) - errPrintln(s" $mod:$ver") + errPrintln(s"Dependencies:\n${indent(Print.dependenciesUnknownConfigs(dependencies))}") if (forceVersions.nonEmpty) { errPrintln("Force versions:") @@ -247,64 +202,29 @@ class Helper( logger.foreach(_.stop()) - if (!res.isDone) { + // FIXME Better to print all the messages related to the exit conditions below, then exit + // rather than exit at the first one + + exitIf(!res.isDone) { errPrintln(s"Maximum number of iteration reached!") sys.exit(1) } - def repr(dep: Dependency) = { - // dep.version can be an interval, whereas the one from project can't - val version = res - .projectCache - .get(dep.moduleVersion) - .map(_._2.version) - .getOrElse(dep.version) - val extra = - if (version == dep.version) "" - else s" ($version for ${dep.version})" - - ( - Seq( - dep.module.organization, - dep.module.name, - dep.attributes.`type` - ) ++ - Some(dep.attributes.classifier) - .filter(_.nonEmpty) - .toSeq ++ - Seq( - version - ) - ).mkString(":") + extra + exitIf(res.errors.nonEmpty) { + s"\n${res.errors.size} error(s):\n" + + res.errors.map { case (dep, errs) => + s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}" + }.mkString("\n") } - val trDeps = res - .minDependencies - .toList - .sortBy(repr) - - if (verbose0 >= 1) { - println("") - println( - trDeps - .map(repr) - .distinct - .mkString("\n") - ) + exitIf(res.conflicts.nonEmpty) { + s"${res.conflicts.size} conflict(s):\n${Print.dependenciesUnknownConfigs(res.conflicts.toVector)}" } - if (res.conflicts.nonEmpty) { - // Needs test - println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") - } + val trDeps = res.minDependencies.toVector - val errors = res.errors - if (errors.nonEmpty) { - println(s"\n${errors.size} error(s):") - for ((dep, errs) <- errors) { - println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") - } - } + if (verbose0 >= 0) + errPrintln(s"Result:\n${indent(Print.dependenciesUnknownConfigs(trDeps))}") def fetch(sources: Boolean, javadoc: Boolean): Seq[File] = { if (verbose0 >= 0) { @@ -354,11 +274,11 @@ class Helper( logger.foreach(_.stop()) - if (errors.nonEmpty) { - println(s"${errors.size} error(s):") - for ((artifact, error) <- errors) { - println(s" ${artifact.url}: $error") - } + exitIf(errors.nonEmpty) { + s"${errors.size} error(s):\n" + + errors.map { case (artifact, error) => + s" ${artifact.url}: $error" + }.mkString("\n") } files0 diff --git a/core/shared/src/main/scala/coursier/util/Config.scala b/core/shared/src/main/scala/coursier/util/Config.scala new file mode 100644 index 000000000..aea3a15b8 --- /dev/null +++ b/core/shared/src/main/scala/coursier/util/Config.scala @@ -0,0 +1,49 @@ +package coursier.util + +import coursier.core.{ Dependency, Resolution } + +object Config { + + // loose attempt at minimizing a set of dependencies from various configs + // `configs` is assumed to be fully unfold + def allDependenciesByConfig( + res: Resolution, + depsByConfig: Map[String, Set[Dependency]], + configs: Map[String, Set[String]] + ): Map[String, Set[Dependency]] = { + + val allDepsByConfig = depsByConfig.map { + case (config, deps) => + config -> res.subset(deps).minDependencies + } + + val filteredAllDepsByConfig = depsByConfig.map { + case (config, allDeps) => + val inherited = configs + .getOrElse(config, Set.empty) + .flatMap(allDepsByConfig.getOrElse(_, Set.empty)) + + config -> (allDeps -- inherited) + } + + filteredAllDepsByConfig + } + + def dependenciesWithConfig( + res: Resolution, + depsByConfig: Map[String, Set[Dependency]], + configs: Map[String, Set[String]] + ): Set[Dependency] = + allDependenciesByConfig(res, depsByConfig, configs) + .flatMap { + case (config, deps) => + deps.map(dep => dep.copy(configuration = s"$config->${dep.configuration}")) + } + .groupBy(_.copy(configuration = "")) + .map { + case (dep, l) => + dep.copy(configuration = l.map(_.configuration).mkString(",")) + } + .toSet + +} diff --git a/core/shared/src/main/scala/coursier/util/Parse.scala b/core/shared/src/main/scala/coursier/util/Parse.scala new file mode 100644 index 000000000..07ec95fc5 --- /dev/null +++ b/core/shared/src/main/scala/coursier/util/Parse.scala @@ -0,0 +1,70 @@ +package coursier.util + +import coursier.core.{Repository, Module} +import coursier.ivy.IvyRepository +import coursier.maven.MavenRepository + +import scala.collection.mutable.ArrayBuffer + +object Parse { + + /** + * Parses coordinates like + * org:name:version + * possibly with attributes, like + * org:name;attr1=val1;attr2=val2:version + */ + def moduleVersion(s: String): Either[String, (Module, String)] = { + + val parts = s.split(":", 3) + + parts match { + case Array(org, rawName, version) => + val splitName = rawName.split(';') + + if (splitName.tail.exists(!_.contains("="))) + Left(s"Malformed attribute in $s") + else { + val name = splitName.head + val attributes = splitName.tail.map(_.split("=", 2)).map { + case Array(key, value) => key -> value + }.toMap + + Right((Module(org, name, attributes), version)) + } + + case _ => + Left(s"Malformed coordinates: $s") + } + } + + /** + * Parses a sequence of coordinates. + * + * @return Sequence of errors, and sequence of modules/versions + */ + def moduleVersions(l: Seq[String]): (Seq[String], Seq[(Module, String)]) = { + + val errors = new ArrayBuffer[String] + val moduleVersions = new ArrayBuffer[(Module, String)] + + for (elem <- l) + moduleVersion(elem) match { + case Left(err) => errors += err + case Right(modVer) => moduleVersions += modVer + } + + (errors.toSeq, moduleVersions.toSeq) + } + + def repository(s: String): Repository = + if (s == "central") + MavenRepository("https://repo1.maven.org/maven2") + else if (s.startsWith("sonatype:")) + MavenRepository(s"https://oss.sonatype.org/content/repositories/${s.stripPrefix("sonatype:")}") + else if (s.startsWith("ivy:")) + IvyRepository(s.stripPrefix("ivy:")) + else + MavenRepository(s) + +} diff --git a/core/shared/src/main/scala/coursier/util/Print.scala b/core/shared/src/main/scala/coursier/util/Print.scala new file mode 100644 index 000000000..6680a8d28 --- /dev/null +++ b/core/shared/src/main/scala/coursier/util/Print.scala @@ -0,0 +1,30 @@ +package coursier.util + +import coursier.core.{ Orders, Dependency } + +object Print { + + def dependency(dep: Dependency): String = + s"${dep.module}:${dep.version}:${dep.configuration}" + + def dependenciesUnknownConfigs(deps: Seq[Dependency]): String = { + + val minDeps = Orders.minDependencies( + deps.toSet, + _ => Map.empty + ) + + val deps0 = minDeps + .groupBy(_.copy(configuration = "")) + .toVector + .map { case (k, l) => + k.copy(configuration = l.toVector.map(_.configuration).sorted.mkString(",")) + } + .sortBy { dep => + (dep.module.organization, dep.module.name, dep.module.toString, dep.version) + } + + deps0.map(dependency).mkString("\n") + } + +} From 9a61d7bbbfec1fccfdc909046ea3c71c5e9fda4b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 10 Jan 2016 21:32:28 +0100 Subject: [PATCH 2/5] Better output of plugin --- .../src/main/scala-2.10/coursier/Tasks.scala | 139 ++++++++---------- 1 file changed, 62 insertions(+), 77 deletions(-) diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index f31eaaae7..fe57b6b29 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -8,6 +8,7 @@ import coursier.core.Publication import coursier.ivy.IvyRepository import coursier.Keys._ import coursier.Structure._ +import coursier.util.{ Config, Print } import org.apache.ivy.core.module.id.ModuleRevisionId import sbt.{ UpdateReport, Classpaths, Resolver, Def } @@ -204,7 +205,7 @@ object Tasks { } def report = { - if (verbosity >= 1) { + if (verbosity >= 2) { println("InterProjectRepository") for (p <- projects) println(s" ${p.module}:${p.version}") @@ -257,14 +258,19 @@ object Tasks { }.sorted.distinct if (verbosity >= 1) { - errPrintln(s"Repositories:") - val repositories0 = repositories.map { - case r: IvyRepository => r.copy(properties = Map.empty) - case r: InterProjectRepository => r.copy(projects = Nil) - case r => r + val repoReprs = repositories.map { + case r: IvyRepository => + s"ivy:${r.pattern}" + case r: InterProjectRepository => + "inter-project" + case r: MavenRepository => + r.root + case r => + // should not happen + r.toString } - for (repo <- repositories0) - errPrintln(s" $repo") + + errPrintln(s"Repositories:\n${repoReprs.map(" "+_).mkString("\n")}") } if (verbosity >= 0) @@ -286,53 +292,58 @@ object Tasks { if (!res.isDone) - throw new Exception(s"Maximum number of iteration reached!") + throw new Exception(s"Maximum number of iteration of dependency resolution reached") + + if (res.conflicts.nonEmpty) { + println(s"${res.conflicts.size} conflict(s):\n ${Print.dependenciesUnknownConfigs(res.conflicts.toVector)}") + throw new Exception(s"Conflict(s) in dependency resolution") + } + + if (res.errors.nonEmpty) { + println(s"\n${res.errors.size} error(s):") + for ((dep, errs) <- res.errors) { + println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") + } + throw new Exception(s"Encountered ${res.errors.length} error(s) in dependency resolution") + } + + val depsByConfig = grouped(currentProject.dependencies) + + val configs = { + val configs0 = ivyConfigurations.value.map { config => + config.name -> config.extendsConfigs.map(_.name) + }.toMap + + def allExtends(c: String) = { + // possibly bad complexity + def helper(current: Set[String]): Set[String] = { + val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) + if ((newSet -- current).nonEmpty) + helper(newSet) + else + newSet + } + + helper(Set(c)) + } + + configs0.map { + case (config, _) => + config -> allExtends(config) + } + } if (verbosity >= 0) errPrintln("Resolution done") - if (verbosity >= 1) - for (depRepr <- depsRepr0(res.minDependencies.toSeq)) - errPrintln(s" $depRepr") + if (verbosity >= 1) { + val finalDeps = Config.dependenciesWithConfig( + res, + depsByConfig.map { case (k, l) => k -> l.toSet }, + configs + ) - def repr(dep: Dependency) = { - // dep.version can be an interval, whereas the one from project can't - val version = res - .projectCache - .get(dep.moduleVersion) - .map(_._2.version) - .getOrElse(dep.version) - val extra = - if (version == dep.version) "" - else s" ($version for ${dep.version})" - - ( - Seq( - dep.module.organization, - dep.module.name, - dep.attributes.`type` - ) ++ - Some(dep.attributes.classifier) - .filter(_.nonEmpty) - .toSeq ++ - Seq( - version - ) - ).mkString(":") + extra - } - - if (res.conflicts.nonEmpty) { - // Needs test - println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") - } - - val errors = res.errors - - if (errors.nonEmpty) { - println(s"\n${errors.size} error(s):") - for ((dep, errs) <- errors) { - println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") - } - throw new Exception(s"Encountered ${errors.length} error(s)") + val repr = Print.dependenciesUnknownConfigs(finalDeps.toVector) + repr.split('\n').map(" "+_).mkString("\n") } val classifiers = @@ -375,30 +386,6 @@ object Tasks { if (verbosity >= 0) errPrintln(s"Fetching artifacts: done") - val configs = { - val configs0 = ivyConfigurations.value.map { config => - config.name -> config.extendsConfigs.map(_.name) - }.toMap - - def allExtends(c: String) = { - // possibly bad complexity - def helper(current: Set[String]): Set[String] = { - val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) - if ((newSet -- current).nonEmpty) - helper(newSet) - else - newSet - } - - helper(Set(c)) - } - - configs0.map { - case (config, _) => - config -> allExtends(config) - } - } - def artifactFileOpt(artifact: Artifact) = { val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded")) @@ -413,8 +400,6 @@ object Tasks { } } - val depsByConfig = grouped(currentProject.dependencies) - writeIvyFiles() ToSbt.updateReport( From f02d26c3bfd8c6f6ee9cbbbb512e31fc1894a0cb Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 10 Jan 2016 21:32:28 +0100 Subject: [PATCH 3/5] Remove dead / erroneous code --- .../scala-2.11/coursier/cli/Coursier.scala | 26 ++----------------- .../main/scala-2.11/coursier/cli/Helper.scala | 2 +- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala index 6c91775dd..67b937455 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala @@ -101,9 +101,6 @@ case class Launch( @Short("M") @Short("main") mainClass: String, - @Short("c") - @Help("Assume coursier is a dependency of the launched app, and share the coursier dependency of the launcher with it - allows the launched app to get the resolution that launched it via ResolutionClassLoader") - addCoursier: Boolean, @Recurse common: CommonOptions ) extends CoursierCommand { @@ -117,27 +114,8 @@ case class Launch( } } - val extraForceVersions = - if (addCoursier) - ??? - else - Seq.empty[String] - - val dontFilterOut = - if (addCoursier) { - val url = classOf[coursier.core.Resolution].getProtectionDomain.getCodeSource.getLocation - - if (url.getProtocol == "file") - Seq(new File(url.getPath)) - else { - Console.err.println(s"Cannot get the location of the JAR of coursier ($url not a file URL)") - sys.exit(255) - } - } else - Seq.empty[File] - val helper = new Helper( - common.copy(forceVersion = common.forceVersion ++ extraForceVersions), + common.copy(forceVersion = common.forceVersion), rawDependencies ) @@ -147,7 +125,7 @@ case class Launch( files0.map(_.toURI.toURL).toArray, new ClasspathFilter( Thread.currentThread().getContextClassLoader, - Coursier.baseCp.map(new File(_)).toSet -- dontFilterOut, + Coursier.baseCp.map(new File(_)).toSet, exclude = true ) ) diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 3959274b6..51b67bcfe 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -58,7 +58,7 @@ object Util { class Helper( common: CommonOptions, - remainingArgs: Seq[String] + rawDependencies: Seq[String] ) { import common._ import Helper.errPrintln From 1ebced021b634d9db781a4e4d94ad626303e1769 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 10 Jan 2016 21:32:29 +0100 Subject: [PATCH 4/5] Add support for isolate ClassLoaders --- .../scala-2.11/coursier/cli/Coursier.scala | 104 ++++++++++++++++-- .../main/scala-2.11/coursier/cli/Helper.scala | 14 ++- .../coursier/cli/IsolatedClassLoader.scala | 19 ++++ 3 files changed, 123 insertions(+), 14 deletions(-) create mode 100644 cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala diff --git a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala index 67b937455..fb14f4eca 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala @@ -101,6 +101,10 @@ case class Launch( @Short("M") @Short("main") mainClass: String, + @ExtraName("I") + isolated: List[String], + @ExtraName("i") + isolateTarget: List[String], @Recurse common: CommonOptions ) extends CoursierCommand { @@ -116,24 +120,102 @@ case class Launch( val helper = new Helper( common.copy(forceVersion = common.forceVersion), - rawDependencies + rawDependencies ++ isolated ) + + // FIXME Some duplication with similar things in Helper + val (splitIsolated, malformedIsolated) = isolated + .toVector + .map(_.split(":", 3).toSeq) + .partition(_.length == 3) + + if (malformedIsolated.nonEmpty) { + if (malformedIsolated.nonEmpty) { + Console.err.println("Malformed dependency(ies), should be like org:name:version") + for (s <- malformedIsolated) + Console.err.println(s" ${s.mkString(":")}") + } + + sys.exit(1) + } + + val isolatedModuleVersions = splitIsolated.map{ + case Seq(org, namePart, version) => + val p = namePart.split(';') + val name = p.head + val splitAttributes = p.tail.map(_.split("=", 2).toSeq).toSeq + val malformedAttributes = splitAttributes.filter(_.length != 2) + if (malformedAttributes.nonEmpty) { + Console.err.println(s"Malformed attributes in ${splitIsolated.mkString(":")}") + // :( + sys.exit(255) + } + val attributes = splitAttributes.collect { + case Seq(k, v) => k -> v + } + (Module(org, name, attributes.toMap), version) + } + + val isolatedDeps = isolatedModuleVersions.map{case (mod, ver) => + Dependency(mod, ver, configuration = "runtime") + } + + + val isolateTargets = { + val l = isolateTarget.flatMap(_.split(',')).filter(_.nonEmpty) + if (l.isEmpty) + Array("default") + else + l.toArray + } + val files0 = helper.fetch(sources = false, javadoc = false) - val cl = new URLClassLoader( - files0.map(_.toURI.toURL).toArray, - new ClasspathFilter( - Thread.currentThread().getContextClassLoader, - Coursier.baseCp.map(new File(_)).toSet, - exclude = true - ) + + val parentLoader0 = new ClasspathFilter( + Thread.currentThread().getContextClassLoader, + Coursier.baseCp.map(new File(_)).toSet, + exclude = true + ) + + val (parentLoader, filteredFiles) = + if (isolated.isEmpty) + (parentLoader0, files0) + else { + // FIXME These were already fetched above + val isolatedFiles = + helper.fetch(sources = false, javadoc = false, subset = isolatedDeps.toSet) + + val isolatedLoader = new IsolatedClassLoader( + isolatedFiles.map(_.toURI.toURL).toArray, + parentLoader0, + isolateTargets + ) + + val filteredFiles0 = files0.filterNot(isolatedFiles.toSet) + + if (common.verbose0 >= 1) { + Console.err.println(s"Isolated loader files:") + for (f <- isolatedFiles.map(_.toString).sorted) + Console.err.println(s" $f") + Console.err.println(s"Remaining files:") + for (f <- filteredFiles0.map(_.toString).sorted) + Console.err.println(s" $f") + } + + (isolatedLoader, filteredFiles0) + } + + val loader = new URLClassLoader( + filteredFiles.map(_.toURI.toURL).toArray, + parentLoader ) val mainClass0 = if (mainClass.nonEmpty) mainClass else { - val mainClasses = Helper.mainClasses(cl) + val mainClasses = Helper.mainClasses(loader) val mainClass = if (mainClasses.isEmpty) { @@ -166,7 +248,7 @@ case class Launch( } val cls = - try cl.loadClass(mainClass0) + try loader.loadClass(mainClass0) catch { case e: ClassNotFoundException => Helper.errPrintln(s"Error: class $mainClass0 not found") sys.exit(255) @@ -183,7 +265,7 @@ case class Launch( else if (common.verbose0 == 0) Helper.errPrintln(s"Launching") - Thread.currentThread().setContextClassLoader(cl) + Thread.currentThread().setContextClassLoader(loader) method.invoke(null, extraArgs.toArray) } diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 51b67bcfe..51ac63ba3 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -226,7 +226,12 @@ class Helper( if (verbose0 >= 0) errPrintln(s"Result:\n${indent(Print.dependenciesUnknownConfigs(trDeps))}") - def fetch(sources: Boolean, javadoc: Boolean): Seq[File] = { + def fetch( + sources: Boolean, + javadoc: Boolean, + subset: Set[Dependency] = null + ): Seq[File] = { + if (verbose0 >= 0) { val msg = cachePolicies match { case Seq(CachePolicy.LocalOnly) => @@ -237,6 +242,9 @@ class Helper( errPrintln(msg) } + + val res0 = Option(subset).fold(res)(res.subset) + val artifacts = if (sources || javadoc) { var classifiers = Seq.empty[String] @@ -245,9 +253,9 @@ class Helper( if (javadoc) classifiers = classifiers :+ "javadoc" - res.classifiersArtifacts(classifiers) + res0.classifiersArtifacts(classifiers) } else - res.artifacts + res0.artifacts val logger = if (verbose0 >= 0) diff --git a/cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala b/cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala new file mode 100644 index 000000000..c07184672 --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala @@ -0,0 +1,19 @@ +package coursier.cli + +import java.net.{ URL, URLClassLoader } + +class IsolatedClassLoader( + urls: Array[URL], + parent: ClassLoader, + isolationTargets: Array[String] +) extends URLClassLoader(urls, parent) { + + /** + * Applications wanting to access an isolated `ClassLoader` should inspect the hierarchy of + * loaders, and look into each of them for this method, by reflection. Then they should + * call it (still by reflection), and look for an agreed in advance target in it. If it is found, + * then the corresponding `ClassLoader` is the one with isolated dependencies. + */ + def getIsolationTargets: Array[String] = isolationTargets + +} From 3259fec276a073de54fdf089829014cb34e2c4d9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 10 Jan 2016 21:32:30 +0100 Subject: [PATCH 5/5] Multiple isolation levels --- .../scala-2.11/coursier/cli/Coursier.scala | 141 ++++++++++-------- .../main/scala-2.11/coursier/cli/Helper.scala | 11 +- 2 files changed, 80 insertions(+), 72 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala index fb14f4eca..28483404f 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala @@ -9,7 +9,7 @@ import java.util.Properties import java.util.zip.{ ZipEntry, ZipOutputStream, ZipInputStream } import caseapp.{ HelpMessage => Help, ValueDescription => Value, ExtraName => Short, _ } -import coursier.util.ClasspathFilter +import coursier.util.{ Parse, ClasspathFilter } case class CommonOptions( @Help("Keep optional dependencies (Maven)") @@ -101,9 +101,11 @@ case class Launch( @Short("M") @Short("main") mainClass: String, - @ExtraName("I") + @Value("target:dependency") + @Short("I") isolated: List[String], - @ExtraName("i") + @Help("Comma-separated isolation targets") + @Short("i") isolateTarget: List[String], @Recurse common: CommonOptions @@ -118,62 +120,65 @@ case class Launch( } } - val helper = new Helper( - common.copy(forceVersion = common.forceVersion), - rawDependencies ++ isolated - ) - - - // FIXME Some duplication with similar things in Helper - val (splitIsolated, malformedIsolated) = isolated - .toVector - .map(_.split(":", 3).toSeq) - .partition(_.length == 3) - - if (malformedIsolated.nonEmpty) { - if (malformedIsolated.nonEmpty) { - Console.err.println("Malformed dependency(ies), should be like org:name:version") - for (s <- malformedIsolated) - Console.err.println(s" ${s.mkString(":")}") - } - - sys.exit(1) - } - - val isolatedModuleVersions = splitIsolated.map{ - case Seq(org, namePart, version) => - val p = namePart.split(';') - val name = p.head - val splitAttributes = p.tail.map(_.split("=", 2).toSeq).toSeq - val malformedAttributes = splitAttributes.filter(_.length != 2) - if (malformedAttributes.nonEmpty) { - Console.err.println(s"Malformed attributes in ${splitIsolated.mkString(":")}") - // :( - sys.exit(255) - } - val attributes = splitAttributes.collect { - case Seq(k, v) => k -> v - } - (Module(org, name, attributes.toMap), version) - } - - val isolatedDeps = isolatedModuleVersions.map{case (mod, ver) => - Dependency(mod, ver, configuration = "runtime") - } - - val isolateTargets = { val l = isolateTarget.flatMap(_.split(',')).filter(_.nonEmpty) - if (l.isEmpty) + val (invalid, valid) = l.partition(_.contains(":")) + if (invalid.nonEmpty) { + Console.err.println(s"Invalid target IDs:") + for (t <- invalid) + Console.err.println(s" $t") + sys.exit(255) + } + if (valid.isEmpty) Array("default") else - l.toArray + valid.toArray } + val (validIsolated, unrecognizedIsolated) = isolated.partition(s => isolateTargets.exists(t => s.startsWith(t + ":"))) + + if (unrecognizedIsolated.nonEmpty) { + Console.err.println(s"Unrecognized isolation targets in:") + for (i <- unrecognizedIsolated) + Console.err.println(s" $i") + sys.exit(255) + } + + val rawIsolated = validIsolated.map { s => + val Array(target, dep) = s.split(":", 2) + target -> dep + } + + val isolatedModuleVersions = rawIsolated.groupBy { case (t, _) => t }.map { + case (t, l) => + val (errors, modVers) = Parse.moduleVersions(l.map { case (_, d) => d }) + + if (errors.nonEmpty) { + errors.foreach(Console.err.println) + sys.exit(255) + } + + t -> modVers + } + + val isolatedDeps = isolatedModuleVersions.map { + case (t, l) => + t -> l.map { + case (mod, ver) => + Dependency(mod, ver, configuration = "runtime") + } + } + + val helper = new Helper( + common.copy(forceVersion = common.forceVersion), + rawDependencies ++ rawIsolated.map { case (_, dep) => dep } + ) + + val files0 = helper.fetch(sources = false, javadoc = false) - val parentLoader0 = new ClasspathFilter( + val parentLoader0: ClassLoader = new ClasspathFilter( Thread.currentThread().getContextClassLoader, Coursier.baseCp.map(new File(_)).toSet, exclude = true @@ -183,22 +188,34 @@ case class Launch( if (isolated.isEmpty) (parentLoader0, files0) else { - // FIXME These were already fetched above - val isolatedFiles = - helper.fetch(sources = false, javadoc = false, subset = isolatedDeps.toSet) + val (isolatedLoader, filteredFiles0) = isolateTargets.foldLeft((parentLoader0, files0)) { + case ((parent, files0), target) => - val isolatedLoader = new IsolatedClassLoader( - isolatedFiles.map(_.toURI.toURL).toArray, - parentLoader0, - isolateTargets - ) + // FIXME These were already fetched above + val isolatedFiles = helper.fetch( + sources = false, + javadoc = false, + subset = isolatedDeps.getOrElse(target, Seq.empty).toSet + ) - val filteredFiles0 = files0.filterNot(isolatedFiles.toSet) + if (common.verbose0 >= 1) { + Console.err.println(s"Isolated loader files:") + for (f <- isolatedFiles.map(_.toString).sorted) + Console.err.println(s" $f") + } + + val isolatedLoader = new IsolatedClassLoader( + isolatedFiles.map(_.toURI.toURL).toArray, + parent, + Array(target) + ) + + val filteredFiles0 = files0.filterNot(isolatedFiles.toSet) + + (isolatedLoader, filteredFiles0) + } if (common.verbose0 >= 1) { - Console.err.println(s"Isolated loader files:") - for (f <- isolatedFiles.map(_.toString).sorted) - Console.err.println(s" $f") Console.err.println(s"Remaining files:") for (f <- filteredFiles0.map(_.toString).sorted) Console.err.println(s" $f") diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 51ac63ba3..62eca141c 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -108,16 +108,7 @@ class Helper( } - val (rawDependencies, extraArgs) = { - val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) - idxOpt.fold((remainingArgs, Seq.empty[String])) { idx => - val (l, r) = remainingArgs.splitAt(idx) - assert(r.nonEmpty) - (l, r.tail) - } - } - - val (modVerErrors, moduleVersions) = Parse.moduleVersions(remainingArgs) + val (modVerErrors, moduleVersions) = Parse.moduleVersions(rawDependencies) prematureExitIf(modVerErrors.nonEmpty) { s"Cannot parse dependencies:\n" + modVerErrors.map(" "+_).mkString("\n")