From 32a4ccc0726fae8bc0c8edb833f8eb39eecdc96f Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:23 +0100 Subject: [PATCH 01/20] Cleaning --- .../src/main/scala/coursier/TermDisplay.scala | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index 65c2cc6a3..a02924d15 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -22,21 +22,21 @@ object Terminal { } else None - class Ansi(val output: Writer) extends AnyVal { + implicit class Ansi(val output: Writer) extends AnyVal { private def control(n: Int, c: Char) = output.write(s"\033[" + n + c) /** * Move up `n` squares */ - def up(n: Int): Unit = if (n == 0) "" else control(n, 'A') + def up(n: Int): Unit = if (n > 0) control(n, 'A') /** * Move down `n` squares */ - def down(n: Int): Unit = if (n == 0) "" else control(n, 'B') + def down(n: Int): Unit = if (n > 0) control(n, 'B') /** * Move left `n` squares */ - def left(n: Int): Unit = if (n == 0) "" else control(n, 'D') + def left(n: Int): Unit = if (n > 0) control(n, 'D') /** * Clear the current line @@ -55,7 +55,8 @@ class TermDisplay( var fallbackMode: Boolean = sys.env.get("COURSIER_NO_TERM").nonEmpty ) extends Cache.Logger { - private val ansi = new Terminal.Ansi(out) + import Terminal.Ansi + private var width = 80 private val refreshInterval = 1000 / 60 private val fallbackRefreshInterval = 1000 @@ -110,7 +111,7 @@ class TermDisplay( def truncatedPrintln(s: String): Unit = { - ansi.clearLine(2) + out.clearLine(2) if (s.length <= width) out.write(s + "\n") @@ -150,7 +151,7 @@ class TermDisplay( assert(info != null, s"Incoherent state ($url)") truncatedPrintln(url) - ansi.clearLine(2) + out.clearLine(2) out.write(s" ${info.display()}\n") } @@ -158,18 +159,18 @@ class TermDisplay( if (displayedCount < lineCount) { for (_ <- 1 to 2; _ <- displayedCount until lineCount) { - ansi.clearLine(2) - ansi.down(1) + out.clearLine(2) + out.down(1) } for (_ <- displayedCount until lineCount) - ansi.up(2) + out.up(2) } for (_ <- downloads0.indices) - ansi.up(2) + out.up(2) - ansi.left(10000) + out.left(10000) out.flush() Thread.sleep(refreshInterval) @@ -221,7 +222,7 @@ class TermDisplay( Terminal.consoleDim("cols") match { case Some(cols) => width = cols - ansi.clearLine(2) + out.clearLine(2) case None => fallbackMode = true } @@ -231,11 +232,11 @@ class TermDisplay( def stop(): Unit = { for (_ <- 1 to 2; _ <- 0 until currentHeight) { - ansi.clearLine(2) - ansi.down(1) + out.clearLine(2) + out.down(1) } for (_ <- 0 until currentHeight) { - ansi.up(2) + out.up(2) } q.put(Left(())) lock.synchronized(()) From 29f8d49c8380c6be25799420a6712308d73fb20d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:23 +0100 Subject: [PATCH 02/20] Switch to scala 2.11.8, SBT scala-js 0.6.7 --- .travis.yml | 4 ++-- appveyor.yml | 4 ++-- build.sbt | 4 ++-- project/plugins.sbt | 2 +- .../com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb1f8452f..d7a69943d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,13 @@ install: os: - osx script: - - project/travis.sh "${TRAVIS_SCALA_VERSION:-2.11.7}" "$TRAVIS_PULL_REQUEST" "$TRAVIS_BRANCH" "$PUBLISH" + - project/travis.sh "${TRAVIS_SCALA_VERSION:-2.11.8}" "$TRAVIS_PULL_REQUEST" "$TRAVIS_BRANCH" "$PUBLISH" # Uncomment once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed # after_success: # - bash <(curl -s https://codecov.io/bash) matrix: include: - - env: TRAVIS_SCALA_VERSION=2.11.7 PUBLISH=1 + - env: TRAVIS_SCALA_VERSION=2.11.8 PUBLISH=1 os: linux jdk: oraclejdk8 - env: TRAVIS_SCALA_VERSION=2.10.6 PUBLISH=1 diff --git a/appveyor.yml b/appveyor.yml index 66d6c946f..e164ab91d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,10 +15,10 @@ install: - cmd: SET SBT_OPTS=-XX:MaxPermSize=2g -Xmx4g - cmd: SET COURSIER_NO_TERM=1 build_script: - - sbt ++2.11.7 clean compile coreJVM/publishLocal + - sbt ++2.11.8 clean compile coreJVM/publishLocal - sbt ++2.10.6 clean compile test_script: - - sbt ++2.11.7 testsJVM/test # Would node be around for testsJS/test? + - sbt ++2.11.8 testsJVM/test # Would node be around for testsJS/test? - sbt ++2.10.6 testsJVM/test cache: - C:\sbt\ diff --git a/build.sbt b/build.sbt index ea1fac52e..06f6dbe38 100644 --- a/build.sbt +++ b/build.sbt @@ -104,8 +104,8 @@ lazy val baseCommonSettings = Seq( ) ++ releaseSettings lazy val commonSettings = baseCommonSettings ++ Seq( - scalaVersion := "2.11.7", - crossScalaVersions := Seq("2.11.7", "2.10.6"), + scalaVersion := "2.11.8", + crossScalaVersions := Seq("2.11.8", "2.10.6"), libraryDependencies ++= { if (scalaVersion.value startsWith "2.10.") Seq(compilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full)) diff --git a/project/plugins.sbt b/project/plugins.sbt index 8ccf88857..a0cfaba4e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,5 @@ addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.6.8") -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.5") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.7") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.1.0") addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.4.0") diff --git a/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT index 65b593655..004454bce 100644 --- a/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT +++ b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT @@ -1,5 +1,5 @@ com.github.alexarchambault:coursier_2.11:1.0.0-SNAPSHOT:compile -org.scala-lang:scala-library:2.11.7:default +org.scala-lang:scala-library:2.11.8:default org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4:default org.scala-lang.modules:scala-xml_2.11:1.0.4:default org.scalaz:scalaz-core_2.11:7.1.2:default From 73e2f5ad68a39b217e951b8203d1eddac210758c Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:23 +0100 Subject: [PATCH 03/20] Add back script to generate standalone launcher *from the sources* The way described in the README uses the artifacts (from Central or Sonatype releases) --- project/generate-standalone-launcher.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 project/generate-standalone-launcher.sh diff --git a/project/generate-standalone-launcher.sh b/project/generate-standalone-launcher.sh new file mode 100755 index 000000000..594adc9d8 --- /dev/null +++ b/project/generate-standalone-launcher.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")/.." + +if [ ! -e cli/target/scala-2.11/proguard/coursier-standalone.jar ]; then + echo "Generating proguarded JAR..." 1>&2 + sbt cli/proguard:proguard +fi + +cat > coursier-standalone << EOF +#!/bin/sh +exec java -noverify -cp "\$0" coursier.cli.Coursier "\$@" +EOF +cat cli/target/scala-2.11/proguard/coursier-standalone.jar >> coursier-standalone +chmod +x coursier-standalone From 5e246762163eeedce972ddf8ed9ad5a9d908cefb Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:24 +0100 Subject: [PATCH 04/20] Add some scaladoc --- .../src/main/scala/coursier/core/Orders.scala | 7 ++--- .../main/scala/coursier/core/Resolution.scala | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Orders.scala b/core/shared/src/main/scala/coursier/core/Orders.scala index 15af6bfe2..fc70b30fd 100644 --- a/core/shared/src/main/scala/coursier/core/Orders.scala +++ b/core/shared/src/main/scala/coursier/core/Orders.scala @@ -35,9 +35,10 @@ object Orders { } /** - * Only relations: - * Compile < Runtime < Test - */ + * Configurations partial order based on configuration mapping `configurations`. + * + * @param configurations: for each configuration, the configurations it directly extends. + */ def configurationPartialOrder(configurations: Map[String, Seq[String]]): PartialOrdering[String] = new PartialOrdering[String] { val allParentsMap = allConfigurations(configurations) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index eb421de65..bcf610cfe 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -847,6 +847,23 @@ final case class Resolution( ) } + /** + * Minimized dependency set. Returns `dependencies` with no redundancy. + * + * E.g. `dependencies` may contains several dependencies towards module org:name:version, + * a first one excluding A and B, and a second one excluding A and C. In practice, B and C will + * be brought anyway, because the first dependency doesn't exclude C, and the second one doesn't + * exclude B. So having both dependencies is equivalent to having only one dependency towards + * org:name:version, excluding just A. + * + * The same kind of substitution / filtering out can be applied with configurations. If + * `dependencies` contains several dependencies towards org:name:version, a first one bringing + * its configuration "runtime", a second one "compile", and the configuration mapping of + * org:name:version says that "runtime" extends "compile", then all the dependencies brought + * by the latter will be brought anyway by the former, so that the latter can be removed. + * + * @return A minimized `dependencies`, applying this kind of substitutions. + */ def minDependencies: Set[Dependency] = Orders.minDependencies( dependencies, @@ -897,6 +914,15 @@ final case class Resolution( .toSeq } yield (dep, err) + /** + * Removes from this `Resolution` dependencies that are not in `dependencies` neither brought + * transitively by them. + * + * This keeps the versions calculated by this `Resolution`. The common dependencies of different + * subsets will thus be guaranteed to have the same versions. + * + * @param dependencies: the dependencies to keep from this `Resolution` + */ def subset(dependencies: Set[Dependency]): Resolution = { val (_, _, finalVersions) = nextDependenciesAndConflicts From 064feb8f3ed3e1f11ffd701a785482ef529e5a95 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:24 +0100 Subject: [PATCH 05/20] Use the right conf mapping separator --- core/shared/src/main/scala/coursier/util/Config.scala | 2 +- core/shared/src/main/scala/coursier/util/Print.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/shared/src/main/scala/coursier/util/Config.scala b/core/shared/src/main/scala/coursier/util/Config.scala index ac6587df4..25fdaae2c 100644 --- a/core/shared/src/main/scala/coursier/util/Config.scala +++ b/core/shared/src/main/scala/coursier/util/Config.scala @@ -42,7 +42,7 @@ object Config { .groupBy(_.copy(configuration = "")) .map { case (dep, l) => - dep.copy(configuration = l.map(_.configuration).mkString(",")) + dep.copy(configuration = l.map(_.configuration).mkString(";")) } .toSet diff --git a/core/shared/src/main/scala/coursier/util/Print.scala b/core/shared/src/main/scala/coursier/util/Print.scala index bbde82229..bdd380b06 100644 --- a/core/shared/src/main/scala/coursier/util/Print.scala +++ b/core/shared/src/main/scala/coursier/util/Print.scala @@ -32,7 +32,7 @@ object Print { .groupBy(_.copy(configuration = "")) .toVector .map { case (k, l) => - k.copy(configuration = l.toVector.map(_.configuration).sorted.mkString(",")) + k.copy(configuration = l.toVector.map(_.configuration).sorted.mkString(";")) } .sortBy { dep => (dep.module.organization, dep.module.name, dep.module.toString, dep.version) From ace927da5b0ed8f32a5a059ee91707316b7db236 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:24 +0100 Subject: [PATCH 06/20] Add development tip about sbt-pack --- README.md | 15 +++++++++++++++ doc/README.md | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/README.md b/README.md index 895a80795..8372ef0e6 100644 --- a/README.md +++ b/README.md @@ -640,6 +640,21 @@ Set `scalaVersion` to `2.10.6` in `build.sbt`. Then re-open / reload the coursie They require `npm install` to have been run once from the `coursier` directory or a subdirectory of it. They can then be run with `sbt testsJS/test`. +#### Quickly running the CLI app from the sources + +Run +``` +$ sbt "~cli/pack" +``` + +This generates and updates a runnable distribution of coursier in `target/pack`, via +the [sbt-pack](https://github.com/xerial/sbt-pack/) plugin. + +It can be run from another terminal with +``` +$ cli/target/pack/bin/coursier +``` + ## Roadmap The first releases were milestones like `0.1.0-M?`. As a launcher, basic Ivy diff --git a/doc/README.md b/doc/README.md index 0e3d04a1b..481682c52 100644 --- a/doc/README.md +++ b/doc/README.md @@ -664,6 +664,21 @@ Set `scalaVersion` to `2.10.6` in `build.sbt`. Then re-open / reload the coursie They require `npm install` to have been run once from the `coursier` directory or a subdirectory of it. They can then be run with `sbt testsJS/test`. +#### Quickly running the CLI app from the sources + +Run +``` +$ sbt "~cli/pack" +``` + +This generates and updates a runnable distribution of coursier in `target/pack`, via +the [sbt-pack](https://github.com/xerial/sbt-pack/) plugin. + +It can be run from another terminal with +``` +$ cli/target/pack/bin/coursier +``` + ## Roadmap The first releases were milestones like `0.1.0-M?`. As a launcher, basic Ivy From d4b2549c133a46738e46fc24596c7cdc4e437eab Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:25 +0100 Subject: [PATCH 07/20] Move things around in cli module --- .../scala-2.11/coursier/cli/Commands.scala | 436 +++++++++++++ .../scala-2.11/coursier/cli/Coursier.scala | 598 +----------------- .../scala-2.11/coursier/cli/Options.scala | 185 ++++++ 3 files changed, 622 insertions(+), 597 deletions(-) create mode 100644 cli/src/main/scala-2.11/coursier/cli/Commands.scala create mode 100644 cli/src/main/scala-2.11/coursier/cli/Options.scala diff --git a/cli/src/main/scala-2.11/coursier/cli/Commands.scala b/cli/src/main/scala-2.11/coursier/cli/Commands.scala new file mode 100644 index 000000000..25ec6c882 --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/Commands.scala @@ -0,0 +1,436 @@ +package coursier +package cli + +import java.io.{ FileInputStream, ByteArrayOutputStream, File, IOException } +import java.net.URLClassLoader +import java.nio.file.{ Files => NIOFiles } +import java.nio.file.attribute.PosixFilePermission +import java.util.Properties +import java.util.zip.{ ZipEntry, ZipOutputStream, ZipInputStream } + +import caseapp.{ HelpMessage => Help, ValueDescription => Value, ExtraName => Short, _ } + +import scala.annotation.tailrec +import scala.language.reflectiveCalls +import scala.util.Try + + +sealed abstract class CoursierCommand extends Command + +case class Resolve( + @Recurse + common: CommonOptions +) extends CoursierCommand { + + // the `val helper = ` part is needed because of DelayedInit it seems + val helper = new Helper(common, remainingArgs, printResultStdout = true) + +} + +case class Fetch( + @Recurse + options: FetchOptions +) extends CoursierCommand { + + val helper = new Helper(options.common, remainingArgs, ignoreErrors = options.force) + + val files0 = helper.fetch(sources = options.sources, javadoc = options.javadoc) + + val out = + if (options.classpath) + files0 + .map(_.toString) + .mkString(File.pathSeparator) + else + files0 + .map(_.toString) + .mkString("\n") + + println(out) + +} + +object Launch { + + @tailrec + def mainClassLoader(cl: ClassLoader): Option[ClassLoader] = + if (cl == null) + None + else { + val isMainLoader = try { + val cl0 = cl.asInstanceOf[Object { + def isBootstrapLoader: Boolean + }] + + cl0.isBootstrapLoader + } catch { + case e: Exception => + false + } + + if (isMainLoader) + Some(cl) + else + mainClassLoader(cl.getParent) + } + +} + +case class Launch( + @Recurse + options: LaunchOptions +) extends CoursierCommand { + + val (rawDependencies, extraArgs) = { + val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) + idxOpt.fold((remainingArgs, Seq.empty[String])) { idx => + val (l, r) = remainingArgs.splitAt(idx) + assert(r.nonEmpty) + (l, r.tail) + } + } + + val helper = new Helper( + options.common, + rawDependencies ++ options.isolated.rawIsolated.map { case (_, dep) => dep } + ) + + + val files0 = helper.fetch(sources = false, javadoc = false) + + val contextLoader = Thread.currentThread().getContextClassLoader + + val parentLoader0: ClassLoader = + if (Try(contextLoader.loadClass("coursier.cli.Launch")).isSuccess) + Launch.mainClassLoader(contextLoader) + .flatMap(cl => Option(cl.getParent)) + .getOrElse { + if (options.common.verbose0 >= 0) + Console.err.println( + "Warning: cannot find the main ClassLoader that launched coursier. " + + "Was coursier launched by its main launcher? " + + "The ClassLoader of the application that is about to be launched will be intertwined " + + "with the one of coursier, which may be a problem if their dependencies conflict." + ) + contextLoader + } + else + // proguarded -> no risk of conflicts, no need to find a specific ClassLoader + contextLoader + + val (parentLoader, filteredFiles) = + if (options.isolated.isolated.isEmpty) + (parentLoader0, files0) + else { + val (isolatedLoader, filteredFiles0) = options.isolated.targets.foldLeft((parentLoader0, files0)) { + case ((parent, files0), target) => + + // FIXME These were already fetched above + val isolatedFiles = helper.fetch( + sources = false, + javadoc = false, + subset = options.isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet + ) + + if (options.common.verbose0 >= 1) { + Console.err.println(s"Isolated loader files:") + for (f <- isolatedFiles.map(_.toString).sorted) + Console.err.println(s" $f") + } + + val isolatedLoader = new IsolatedClassLoader( + isolatedFiles.map(_.toURI.toURL).toArray, + parent, + Array(target) + ) + + val filteredFiles0 = files0.filterNot(isolatedFiles.toSet) + + (isolatedLoader, filteredFiles0) + } + + if (options.common.verbose0 >= 1) { + Console.err.println(s"Remaining files:") + for (f <- filteredFiles0.map(_.toString).sorted) + Console.err.println(s" $f") + } + + (isolatedLoader, filteredFiles0) + } + + val loader = new URLClassLoader( + filteredFiles.map(_.toURI.toURL).toArray, + parentLoader + ) + + val mainClass0 = + if (options.mainClass.nonEmpty) options.mainClass + else { + val mainClasses = Helper.mainClasses(loader) + + val mainClass = + if (mainClasses.isEmpty) { + Helper.errPrintln("No main class found. Specify one with -M or --main.") + sys.exit(255) + } else if (mainClasses.size == 1) { + val (_, mainClass) = mainClasses.head + mainClass + } else { + // Trying to get the main class of the first artifact + val mainClassOpt = for { + (module, _, _) <- helper.moduleVersionConfigs.headOption + mainClass <- mainClasses.collectFirst { + case ((org, name), mainClass) + if org == module.organization && ( + module.name == name || + module.name.startsWith(name + "_") // Ignore cross version suffix + ) => + mainClass + } + } yield mainClass + + mainClassOpt.getOrElse { + Helper.errPrintln(s"Cannot find default main class. Specify one with -M or --main.") + sys.exit(255) + } + } + + mainClass + } + + val cls = + try loader.loadClass(mainClass0) + catch { case e: ClassNotFoundException => + Helper.errPrintln(s"Error: class $mainClass0 not found") + sys.exit(255) + } + val method = + try cls.getMethod("main", classOf[Array[String]]) + catch { case e: NoSuchMethodException => + Helper.errPrintln(s"Error: method main not found in $mainClass0") + sys.exit(255) + } + + if (options.common.verbose0 >= 1) + Helper.errPrintln(s"Launching $mainClass0 ${extraArgs.mkString(" ")}") + else if (options.common.verbose0 == 0) + Helper.errPrintln(s"Launching") + + Thread.currentThread().setContextClassLoader(loader) + method.invoke(null, extraArgs.toArray) +} + +case class Bootstrap( + @Recurse + options: BootstrapOptions +) extends CoursierCommand { + + import scala.collection.JavaConverters._ + + if (options.mainClass.isEmpty) { + Console.err.println(s"Error: no main class specified. Specify one with -M or --main") + sys.exit(255) + } + + if (!options.standalone && options.downloadDir.isEmpty) { + Console.err.println(s"Error: no download dir specified. Specify one with -D or --download-dir") + Console.err.println("E.g. -D \"\\$HOME/.app-name/jars\"") + sys.exit(255) + } + + val (validProperties, wrongProperties) = options.property.partition(_.contains("=")) + if (wrongProperties.nonEmpty) { + Console.err.println(s"Wrong -P / --property option(s):\n${wrongProperties.mkString("\n")}") + sys.exit(255) + } + + val properties0 = validProperties.map { s => + val idx = s.indexOf('=') + assert(idx >= 0) + (s.take(idx), s.drop(idx + 1)) + } + + val bootstrapJar = + Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { + case Some(is) => Cache.readFullySync(is) + case None => + Console.err.println(s"Error: bootstrap JAR not found") + sys.exit(1) + } + + val output0 = new File(options.output) + if (!options.force && output0.exists()) { + Console.err.println(s"Error: ${options.output} already exists, use -f option to force erasing it.") + sys.exit(1) + } + + def zipEntries(zipStream: ZipInputStream): Iterator[(ZipEntry, Array[Byte])] = + new Iterator[(ZipEntry, Array[Byte])] { + var nextEntry = Option.empty[ZipEntry] + def update() = + nextEntry = Option(zipStream.getNextEntry) + + update() + + def hasNext = nextEntry.nonEmpty + def next() = { + val ent = nextEntry.get + val data = Platform.readFullySync(zipStream) + + update() + + (ent, data) + } + } + + + val helper = new Helper(options.common, remainingArgs) + + val (_, isolatedArtifactFiles) = + options.isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) { + case ((done, acc), target) => + val subRes = helper.res.subset(options.isolated.isolatedDeps.getOrElse(target, Nil).toSet) + val subArtifacts = subRes.artifacts.map(_.url) + + val filteredSubArtifacts = subArtifacts.diff(done) + + def subFiles0 = helper.fetch( + sources = false, + javadoc = false, + subset = options.isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet + ) + + val (subUrls, subFiles) = + if (options.standalone) + (Nil, subFiles0) + else + (filteredSubArtifacts, Nil) + + val updatedAcc = acc + (target -> (subUrls, subFiles)) + + (done ++ filteredSubArtifacts, updatedAcc) + } + + val (urls, files) = + if (options.standalone) + ( + Seq.empty[String], + helper.fetch(sources = false, javadoc = false) + ) + else + ( + helper.artifacts(sources = false, javadoc = false).map(_.url), + Seq.empty[File] + ) + + val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v } + val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v } + + val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://")) + if (nonHttpUrls.nonEmpty) + Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}") + + val buffer = new ByteArrayOutputStream() + + val bootstrapZip = new ZipInputStream(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) + val outputZip = new ZipOutputStream(buffer) + + for ((ent, data) <- zipEntries(bootstrapZip)) { + outputZip.putNextEntry(ent) + outputZip.write(data) + outputZip.closeEntry() + } + + + val time = System.currentTimeMillis() + + def putStringEntry(name: String, content: String): Unit = { + val entry = new ZipEntry(name) + entry.setTime(time) + + outputZip.putNextEntry(entry) + outputZip.write(content.getBytes("UTF-8")) + outputZip.closeEntry() + } + + def putEntryFromFile(name: String, f: File): Unit = { + val entry = new ZipEntry(name) + entry.setTime(f.lastModified()) + + outputZip.putNextEntry(entry) + outputZip.write(Cache.readFullySync(new FileInputStream(f))) + outputZip.closeEntry() + } + + putStringEntry("bootstrap-jar-urls", urls.mkString("\n")) + + if (options.isolated.anyIsolatedDep) { + putStringEntry("bootstrap-isolation-ids", options.isolated.targets.mkString("\n")) + + for (target <- options.isolated.targets) { + val urls = isolatedUrls.getOrElse(target, Nil) + val files = isolatedFiles.getOrElse(target, Nil) + putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n")) + putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n")) + } + } + + def pathFor(f: File) = s"jars/${f.getName}" + + for (f <- files) + putEntryFromFile(pathFor(f), f) + + putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n")) + + val propsEntry = new ZipEntry("bootstrap.properties") + propsEntry.setTime(time) + + val properties = new Properties() + properties.setProperty("bootstrap.mainClass", options.mainClass) + if (!options.standalone) + properties.setProperty("bootstrap.jarDir", options.downloadDir) + + outputZip.putNextEntry(propsEntry) + properties.store(outputZip, "") + outputZip.closeEntry() + + outputZip.close() + + // escaping of javaOpt possibly a bit loose :-| + val shellPreamble = Seq( + "#!/usr/bin/env sh", + "exec java -jar " + options.javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\"" + ).mkString("", "\n", "\n") + + try NIOFiles.write(output0.toPath, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) + catch { case e: IOException => + Console.err.println(s"Error while writing $output0: ${e.getMessage}") + sys.exit(1) + } + + + try { + val perms = NIOFiles.getPosixFilePermissions(output0.toPath).asScala.toSet + + var newPerms = perms + if (perms(PosixFilePermission.OWNER_READ)) + newPerms += PosixFilePermission.OWNER_EXECUTE + if (perms(PosixFilePermission.GROUP_READ)) + newPerms += PosixFilePermission.GROUP_EXECUTE + if (perms(PosixFilePermission.OTHERS_READ)) + newPerms += PosixFilePermission.OTHERS_EXECUTE + + if (newPerms != perms) + NIOFiles.setPosixFilePermissions( + output0.toPath, + newPerms.asJava + ) + } catch { + case e: UnsupportedOperationException => + // Ignored + case e: IOException => + Console.err.println(s"Error while making $output0 executable: ${e.getMessage}") + sys.exit(1) + } + +} diff --git a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala index 8e4e2d85b..b9801fba1 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala @@ -1,603 +1,7 @@ package coursier package cli -import java.io.{ FileInputStream, ByteArrayOutputStream, File, IOException } -import java.net.URLClassLoader -import java.nio.file.{ Files => NIOFiles } -import java.nio.file.attribute.PosixFilePermission -import java.util.Properties -import java.util.zip.{ ZipEntry, ZipOutputStream, ZipInputStream } - -import caseapp.{ HelpMessage => Help, ValueDescription => Value, ExtraName => Short, _ } -import coursier.util.Parse - -import scala.annotation.tailrec -import scala.language.reflectiveCalls -import scala.util.Try - -case class CommonOptions( - @Help("Keep optional dependencies (Maven)") - keepOptional: Boolean, - @Help("Download mode (default: missing, that is fetch things missing from cache)") - @Value("offline|update-changing|update|missing|force") - @Short("m") - mode: String = "default", - @Help("Quiet output") - @Short("q") - quiet: Boolean, - @Help("Increase verbosity (specify several times to increase more)") - @Short("v") - verbose: List[Unit], - @Help("Maximum number of resolution iterations (specify a negative value for unlimited, default: 100)") - @Short("N") - maxIterations: Int = 100, - @Help("Repositories - for multiple repositories, separate with comma and/or repeat this option (e.g. -r central,ivy2local -r sonatype-snapshots, or equivalently -r central,ivy2local,sonatype-snapshots)") - @Short("r") - repository: List[String], - @Help("Do not add default repositories (~/.ivy2/local, and Central)") - noDefault: Boolean = false, - @Help("Modify names in Maven repository paths for SBT plugins") - sbtPluginHack: Boolean = false, - @Help("Drop module attributes starting with 'info.' - these are sometimes used by projects built with SBT") - dropInfoAttr: Boolean = false, - @Help("Force module version") - @Value("organization:name:forcedVersion") - @Short("V") - forceVersion: List[String], - @Help("Exclude module") - @Value("organization:name") - @Short("E") - exclude: List[String], - @Help("Consider provided dependencies to be intransitive. Applies to all the provided dependencies.") - intransitive: Boolean, - @Help("Classifiers that should be fetched") - @Value("classifier1,classifier2,...") - @Short("C") - classifier: List[String], - @Help("Default configuration (default(compile) by default)") - @Value("configuration") - @Short("c") - defaultConfiguration: String = "default(compile)", - @Help("Maximum number of parallel downloads (default: 6)") - @Short("n") - parallel: Int = 6, - @Help("Checksums") - @Value("checksum1,checksum2,... - end with none to allow for no checksum validation if none are available") - checksum: List[String], - @Recurse - cacheOptions: CacheOptions -) { - val verbose0 = verbose.length - (if (quiet) 1 else 0) - lazy val classifier0 = classifier.flatMap(_.split(',')).filter(_.nonEmpty) -} - -case class CacheOptions( - @Help("Cache directory (defaults to environment variable COURSIER_CACHE or ~/.coursier/cache/v1)") - @Short("C") - cache: String = Cache.default.toString -) - -sealed abstract class CoursierCommand extends Command - -case class Resolve( - @Recurse - common: CommonOptions -) extends CoursierCommand { - - // the `val helper = ` part is needed because of DelayedInit it seems - val helper = new Helper(common, remainingArgs, printResultStdout = true) - -} - -case class Fetch( - @Help("Fetch source artifacts") - @Short("S") - sources: Boolean, - @Help("Fetch javadoc artifacts") - @Short("D") - javadoc: Boolean, - @Help("Print java -cp compatible output") - @Short("p") - classpath: Boolean, - @Help("Fetch artifacts even if the resolution is errored") - force: Boolean, - @Recurse - common: CommonOptions -) extends CoursierCommand { - - val helper = new Helper(common, remainingArgs, ignoreErrors = force) - - val files0 = helper.fetch(sources = sources, javadoc = javadoc) - - val out = - if (classpath) - files0 - .map(_.toString) - .mkString(File.pathSeparator) - else - files0 - .map(_.toString) - .mkString("\n") - - println(out) - -} - -case class IsolatedLoaderOptions( - @Value("target:dependency") - @Short("I") - isolated: List[String], - @Help("Comma-separated isolation targets") - @Short("i") - isolateTarget: List[String] -) { - - def anyIsolatedDep = isolateTarget.nonEmpty || isolated.nonEmpty - - lazy val targets = { - val l = isolateTarget.flatMap(_.split(',')).filter(_.nonEmpty) - val (invalid, valid) = l.partition(_.contains(":")) - if (invalid.nonEmpty) { - Console.err.println(s"Invalid target IDs:") - for (t <- invalid) - Console.err.println(s" $t") - sys.exit(255) - } - if (valid.isEmpty) - Array("default") - else - valid.toArray - } - - lazy val (validIsolated, unrecognizedIsolated) = isolated.partition(s => targets.exists(t => s.startsWith(t + ":"))) - - def check() = { - if (unrecognizedIsolated.nonEmpty) { - Console.err.println(s"Unrecognized isolation targets in:") - for (i <- unrecognizedIsolated) - Console.err.println(s" $i") - sys.exit(255) - } - } - - lazy val rawIsolated = validIsolated.map { s => - val Array(target, dep) = s.split(":", 2) - target -> dep - } - - lazy val isolatedModuleVersions = rawIsolated.groupBy { case (t, _) => t }.map { - case (t, l) => - val (errors, modVers) = Parse.moduleVersions(l.map { case (_, d) => d }) - - if (errors.nonEmpty) { - errors.foreach(Console.err.println) - sys.exit(255) - } - - t -> modVers - } - - lazy val isolatedDeps = isolatedModuleVersions.map { - case (t, l) => - t -> l.map { - case (mod, ver) => - Dependency(mod, ver, configuration = "runtime") - } - } - -} - -object Launch { - - @tailrec - def mainClassLoader(cl: ClassLoader): Option[ClassLoader] = - if (cl == null) - None - else { - val isMainLoader = try { - val cl0 = cl.asInstanceOf[Object { - def isBootstrapLoader: Boolean - }] - - cl0.isBootstrapLoader - } catch { - case e: Exception => - false - } - - if (isMainLoader) - Some(cl) - else - mainClassLoader(cl.getParent) - } - -} - -case class Launch( - @Short("M") - @Short("main") - mainClass: String, - @Recurse - isolated: IsolatedLoaderOptions, - @Recurse - common: CommonOptions -) extends CoursierCommand { - - val (rawDependencies, extraArgs) = { - val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) - idxOpt.fold((remainingArgs, Seq.empty[String])) { idx => - val (l, r) = remainingArgs.splitAt(idx) - assert(r.nonEmpty) - (l, r.tail) - } - } - - val helper = new Helper( - common, - rawDependencies ++ isolated.rawIsolated.map { case (_, dep) => dep } - ) - - - val files0 = helper.fetch(sources = false, javadoc = false) - - val contextLoader = Thread.currentThread().getContextClassLoader - - val parentLoader0: ClassLoader = - if (Try(contextLoader.loadClass("coursier.cli.Launch")).isSuccess) - Launch.mainClassLoader(contextLoader) - .flatMap(cl => Option(cl.getParent)) - .getOrElse { - if (common.verbose0 >= 0) - Console.err.println( - "Warning: cannot find the main ClassLoader that launched coursier. " + - "Was coursier launched by its main launcher? " + - "The ClassLoader of the application that is about to be launched will be intertwined " + - "with the one of coursier, which may be a problem if their dependencies conflict." - ) - contextLoader - } - else - // proguarded -> no risk of conflicts, no need to find a specific ClassLoader - contextLoader - - val (parentLoader, filteredFiles) = - if (isolated.isolated.isEmpty) - (parentLoader0, files0) - else { - val (isolatedLoader, filteredFiles0) = isolated.targets.foldLeft((parentLoader0, files0)) { - case ((parent, files0), target) => - - // FIXME These were already fetched above - val isolatedFiles = helper.fetch( - sources = false, - javadoc = false, - subset = isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet - ) - - if (common.verbose0 >= 1) { - Console.err.println(s"Isolated loader files:") - for (f <- isolatedFiles.map(_.toString).sorted) - Console.err.println(s" $f") - } - - val isolatedLoader = new IsolatedClassLoader( - isolatedFiles.map(_.toURI.toURL).toArray, - parent, - Array(target) - ) - - val filteredFiles0 = files0.filterNot(isolatedFiles.toSet) - - (isolatedLoader, filteredFiles0) - } - - if (common.verbose0 >= 1) { - Console.err.println(s"Remaining files:") - for (f <- filteredFiles0.map(_.toString).sorted) - Console.err.println(s" $f") - } - - (isolatedLoader, filteredFiles0) - } - - val loader = new URLClassLoader( - filteredFiles.map(_.toURI.toURL).toArray, - parentLoader - ) - - val mainClass0 = - if (mainClass.nonEmpty) mainClass - else { - val mainClasses = Helper.mainClasses(loader) - - val mainClass = - if (mainClasses.isEmpty) { - Helper.errPrintln("No main class found. Specify one with -M or --main.") - sys.exit(255) - } else if (mainClasses.size == 1) { - val (_, mainClass) = mainClasses.head - mainClass - } else { - // Trying to get the main class of the first artifact - val mainClassOpt = for { - (module, _, _) <- helper.moduleVersionConfigs.headOption - mainClass <- mainClasses.collectFirst { - case ((org, name), mainClass) - if org == module.organization && ( - module.name == name || - module.name.startsWith(name + "_") // Ignore cross version suffix - ) => - mainClass - } - } yield mainClass - - mainClassOpt.getOrElse { - Helper.errPrintln(s"Cannot find default main class. Specify one with -M or --main.") - sys.exit(255) - } - } - - mainClass - } - - val cls = - try loader.loadClass(mainClass0) - catch { case e: ClassNotFoundException => - Helper.errPrintln(s"Error: class $mainClass0 not found") - sys.exit(255) - } - val method = - try cls.getMethod("main", classOf[Array[String]]) - catch { case e: NoSuchMethodException => - Helper.errPrintln(s"Error: method main not found in $mainClass0") - sys.exit(255) - } - - if (common.verbose0 >= 1) - Helper.errPrintln(s"Launching $mainClass0 ${extraArgs.mkString(" ")}") - else if (common.verbose0 == 0) - Helper.errPrintln(s"Launching") - - Thread.currentThread().setContextClassLoader(loader) - method.invoke(null, extraArgs.toArray) -} - -case class Bootstrap( - @Short("M") - @Short("main") - mainClass: String, - @Short("o") - output: String = "bootstrap", - @Short("D") - downloadDir: String, - @Short("f") - force: Boolean, - @Help("Generate a standalone launcher, with all JARs included, instead of one downloading its dependencies on startup.") - @Short("s") - standalone: Boolean, - @Help("Set Java properties in the generated launcher.") - @Value("key=value") - @Short("P") - property: List[String], - @Help("Set Java command-line options in the generated launcher.") - @Value("option") - @Short("J") - javaOpt: List[String], - @Recurse - isolated: IsolatedLoaderOptions, - @Recurse - common: CommonOptions -) extends CoursierCommand { - - import scala.collection.JavaConverters._ - - if (mainClass.isEmpty) { - Console.err.println(s"Error: no main class specified. Specify one with -M or --main") - sys.exit(255) - } - - if (!standalone && downloadDir.isEmpty) { - Console.err.println(s"Error: no download dir specified. Specify one with -D or --download-dir") - Console.err.println("E.g. -D \"\\$HOME/.app-name/jars\"") - sys.exit(255) - } - - val (validProperties, wrongProperties) = property.partition(_.contains("=")) - if (wrongProperties.nonEmpty) { - Console.err.println(s"Wrong -P / --property option(s):\n${wrongProperties.mkString("\n")}") - sys.exit(255) - } - - val properties0 = validProperties.map { s => - val idx = s.indexOf('=') - assert(idx >= 0) - (s.take(idx), s.drop(idx + 1)) - } - - val bootstrapJar = - Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { - case Some(is) => Cache.readFullySync(is) - case None => - Console.err.println(s"Error: bootstrap JAR not found") - sys.exit(1) - } - - val output0 = new File(output) - if (!force && output0.exists()) { - Console.err.println(s"Error: $output already exists, use -f option to force erasing it.") - sys.exit(1) - } - - def zipEntries(zipStream: ZipInputStream): Iterator[(ZipEntry, Array[Byte])] = - new Iterator[(ZipEntry, Array[Byte])] { - var nextEntry = Option.empty[ZipEntry] - def update() = - nextEntry = Option(zipStream.getNextEntry) - - update() - - def hasNext = nextEntry.nonEmpty - def next() = { - val ent = nextEntry.get - val data = Platform.readFullySync(zipStream) - - update() - - (ent, data) - } - } - - - val helper = new Helper(common, remainingArgs) - - val (_, isolatedArtifactFiles) = - isolated.targets.foldLeft((Vector.empty[String], Map.empty[String, (Seq[String], Seq[File])])) { - case ((done, acc), target) => - val subRes = helper.res.subset(isolated.isolatedDeps.getOrElse(target, Nil).toSet) - val subArtifacts = subRes.artifacts.map(_.url) - - val filteredSubArtifacts = subArtifacts.diff(done) - - def subFiles0 = helper.fetch( - sources = false, - javadoc = false, - subset = isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet - ) - - val (subUrls, subFiles) = - if (standalone) - (Nil, subFiles0) - else - (filteredSubArtifacts, Nil) - - val updatedAcc = acc + (target -> (subUrls, subFiles)) - - (done ++ filteredSubArtifacts, updatedAcc) - } - - val (urls, files) = - if (standalone) - ( - Seq.empty[String], - helper.fetch(sources = false, javadoc = false) - ) - else - ( - helper.artifacts(sources = false, javadoc = false).map(_.url), - Seq.empty[File] - ) - - val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v } - val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v } - - val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://")) - if (nonHttpUrls.nonEmpty) - Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}") - - val buffer = new ByteArrayOutputStream() - - val bootstrapZip = new ZipInputStream(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) - val outputZip = new ZipOutputStream(buffer) - - for ((ent, data) <- zipEntries(bootstrapZip)) { - outputZip.putNextEntry(ent) - outputZip.write(data) - outputZip.closeEntry() - } - - - val time = System.currentTimeMillis() - - def putStringEntry(name: String, content: String): Unit = { - val entry = new ZipEntry(name) - entry.setTime(time) - - outputZip.putNextEntry(entry) - outputZip.write(content.getBytes("UTF-8")) - outputZip.closeEntry() - } - - def putEntryFromFile(name: String, f: File): Unit = { - val entry = new ZipEntry(name) - entry.setTime(f.lastModified()) - - outputZip.putNextEntry(entry) - outputZip.write(Cache.readFullySync(new FileInputStream(f))) - outputZip.closeEntry() - } - - putStringEntry("bootstrap-jar-urls", urls.mkString("\n")) - - if (isolated.anyIsolatedDep) { - putStringEntry("bootstrap-isolation-ids", isolated.targets.mkString("\n")) - - for (target <- isolated.targets) { - val urls = isolatedUrls.getOrElse(target, Nil) - val files = isolatedFiles.getOrElse(target, Nil) - putStringEntry(s"bootstrap-isolation-$target-jar-urls", urls.mkString("\n")) - putStringEntry(s"bootstrap-isolation-$target-jar-resources", files.map(pathFor).mkString("\n")) - } - } - - def pathFor(f: File) = s"jars/${f.getName}" - - for (f <- files) - putEntryFromFile(pathFor(f), f) - - putStringEntry("bootstrap-jar-resources", files.map(pathFor).mkString("\n")) - - val propsEntry = new ZipEntry("bootstrap.properties") - propsEntry.setTime(time) - - val properties = new Properties() - properties.setProperty("bootstrap.mainClass", mainClass) - if (!standalone) - properties.setProperty("bootstrap.jarDir", downloadDir) - - outputZip.putNextEntry(propsEntry) - properties.store(outputZip, "") - outputZip.closeEntry() - - outputZip.close() - - // escaping of javaOpt possibly a bit loose :-| - val shellPreamble = Seq( - "#!/usr/bin/env sh", - "exec java -jar " + javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\"" - ).mkString("", "\n", "\n") - - try NIOFiles.write(output0.toPath, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) - catch { case e: IOException => - Console.err.println(s"Error while writing $output0: ${e.getMessage}") - sys.exit(1) - } - - - try { - val perms = NIOFiles.getPosixFilePermissions(output0.toPath).asScala.toSet - - var newPerms = perms - if (perms(PosixFilePermission.OWNER_READ)) - newPerms += PosixFilePermission.OWNER_EXECUTE - if (perms(PosixFilePermission.GROUP_READ)) - newPerms += PosixFilePermission.GROUP_EXECUTE - if (perms(PosixFilePermission.OTHERS_READ)) - newPerms += PosixFilePermission.OTHERS_EXECUTE - - if (newPerms != perms) - NIOFiles.setPosixFilePermissions( - output0.toPath, - newPerms.asJava - ) - } catch { - case e: UnsupportedOperationException => - // Ignored - case e: IOException => - Console.err.println(s"Error while making $output0 executable: ${e.getMessage}") - sys.exit(1) - } - -} +import caseapp._ object Coursier extends CommandAppOf[CoursierCommand] { override def appName = "Coursier" diff --git a/cli/src/main/scala-2.11/coursier/cli/Options.scala b/cli/src/main/scala-2.11/coursier/cli/Options.scala new file mode 100644 index 000000000..b9238010b --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/Options.scala @@ -0,0 +1,185 @@ +package coursier +package cli + +import caseapp.{ HelpMessage => Help, ValueDescription => Value, ExtraName => Short, _ } + +import coursier.util.Parse + +case class CommonOptions( + @Help("Keep optional dependencies (Maven)") + keepOptional: Boolean, + @Help("Download mode (default: missing, that is fetch things missing from cache)") + @Value("offline|update-changing|update|missing|force") + @Short("m") + mode: String = "default", + @Help("Quiet output") + @Short("q") + quiet: Boolean, + @Help("Increase verbosity (specify several times to increase more)") + @Short("v") + verbose: List[Unit], + @Help("Maximum number of resolution iterations (specify a negative value for unlimited, default: 100)") + @Short("N") + maxIterations: Int = 100, + @Help("Repositories - for multiple repositories, separate with comma and/or repeat this option (e.g. -r central,ivy2local -r sonatype-snapshots, or equivalently -r central,ivy2local,sonatype-snapshots)") + @Short("r") + repository: List[String], + @Help("Do not add default repositories (~/.ivy2/local, and Central)") + noDefault: Boolean = false, + @Help("Modify names in Maven repository paths for SBT plugins") + sbtPluginHack: Boolean = false, + @Help("Drop module attributes starting with 'info.' - these are sometimes used by projects built with SBT") + dropInfoAttr: Boolean = false, + @Help("Force module version") + @Value("organization:name:forcedVersion") + @Short("V") + forceVersion: List[String], + @Help("Exclude module") + @Value("organization:name") + @Short("E") + exclude: List[String], + @Help("Consider provided dependencies to be intransitive. Applies to all the provided dependencies.") + intransitive: Boolean, + @Help("Classifiers that should be fetched") + @Value("classifier1,classifier2,...") + @Short("C") + classifier: List[String], + @Help("Default configuration (default(compile) by default)") + @Value("configuration") + @Short("c") + defaultConfiguration: String = "default(compile)", + @Help("Maximum number of parallel downloads (default: 6)") + @Short("n") + parallel: Int = 6, + @Help("Checksums") + @Value("checksum1,checksum2,... - end with none to allow for no checksum validation if none are available") + checksum: List[String], + @Recurse + cacheOptions: CacheOptions +) { + val verbose0 = verbose.length - (if (quiet) 1 else 0) + lazy val classifier0 = classifier.flatMap(_.split(',')).filter(_.nonEmpty) +} + +case class CacheOptions( + @Help("Cache directory (defaults to environment variable COURSIER_CACHE or ~/.coursier/cache/v1)") + @Short("C") + cache: String = Cache.default.toString +) + +case class IsolatedLoaderOptions( + @Value("target:dependency") + @Short("I") + isolated: List[String], + @Help("Comma-separated isolation targets") + @Short("i") + isolateTarget: List[String] +) { + + def anyIsolatedDep = isolateTarget.nonEmpty || isolated.nonEmpty + + lazy val targets = { + val l = isolateTarget.flatMap(_.split(',')).filter(_.nonEmpty) + val (invalid, valid) = l.partition(_.contains(":")) + if (invalid.nonEmpty) { + Console.err.println(s"Invalid target IDs:") + for (t <- invalid) + Console.err.println(s" $t") + sys.exit(255) + } + if (valid.isEmpty) + Array("default") + else + valid.toArray + } + + lazy val (validIsolated, unrecognizedIsolated) = isolated.partition(s => targets.exists(t => s.startsWith(t + ":"))) + + def check() = { + if (unrecognizedIsolated.nonEmpty) { + Console.err.println(s"Unrecognized isolation targets in:") + for (i <- unrecognizedIsolated) + Console.err.println(s" $i") + sys.exit(255) + } + } + + lazy val rawIsolated = validIsolated.map { s => + val Array(target, dep) = s.split(":", 2) + target -> dep + } + + lazy val isolatedModuleVersions = rawIsolated.groupBy { case (t, _) => t }.map { + case (t, l) => + val (errors, modVers) = Parse.moduleVersions(l.map { case (_, d) => d }) + + if (errors.nonEmpty) { + errors.foreach(Console.err.println) + sys.exit(255) + } + + t -> modVers + } + + lazy val isolatedDeps = isolatedModuleVersions.map { + case (t, l) => + t -> l.map { + case (mod, ver) => + Dependency(mod, ver, configuration = "runtime") + } + } + +} + +case class FetchOptions( + @Help("Fetch source artifacts") + @Short("S") + sources: Boolean, + @Help("Fetch javadoc artifacts") + @Short("D") + javadoc: Boolean, + @Help("Print java -cp compatible output") + @Short("p") + classpath: Boolean, + @Help("Fetch artifacts even if the resolution is errored") + force: Boolean, + @Recurse + common: CommonOptions +) + +case class LaunchOptions( + @Short("M") + @Short("main") + mainClass: String, + @Recurse + isolated: IsolatedLoaderOptions, + @Recurse + common: CommonOptions +) + +case class BootstrapOptions( + @Short("M") + @Short("main") + mainClass: String, + @Short("o") + output: String = "bootstrap", + @Short("D") + downloadDir: String, + @Short("f") + force: Boolean, + @Help("Generate a standalone launcher, with all JARs included, instead of one downloading its dependencies on startup.") + @Short("s") + standalone: Boolean, + @Help("Set Java properties in the generated launcher.") + @Value("key=value") + @Short("P") + property: List[String], + @Help("Set Java command-line options in the generated launcher.") + @Value("option") + @Short("J") + javaOpt: List[String], + @Recurse + isolated: IsolatedLoaderOptions, + @Recurse + common: CommonOptions +) From 4be4f761a6415fa3eb449f3a55868627bc4f3ee6 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:25 +0100 Subject: [PATCH 08/20] Keep moving things around in cli module In particular, put each command in a separate file... --- .../cli/{Commands.scala => Bootstrap.scala} | 223 +----------------- .../scala-2.11/coursier/cli/Coursier.scala | 31 ++- .../main/scala-2.11/coursier/cli/Fetch.scala | 31 +++ .../coursier/cli/IsolatedClassLoader.scala | 19 -- .../main/scala-2.11/coursier/cli/Launch.scala | 196 +++++++++++++++ .../scala-2.11/coursier/cli/Resolve.scala | 14 ++ 6 files changed, 277 insertions(+), 237 deletions(-) rename cli/src/main/scala-2.11/coursier/cli/{Commands.scala => Bootstrap.scala} (51%) create mode 100644 cli/src/main/scala-2.11/coursier/cli/Fetch.scala delete mode 100644 cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala create mode 100644 cli/src/main/scala-2.11/coursier/cli/Launch.scala create mode 100644 cli/src/main/scala-2.11/coursier/cli/Resolve.scala diff --git a/cli/src/main/scala-2.11/coursier/cli/Commands.scala b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala similarity index 51% rename from cli/src/main/scala-2.11/coursier/cli/Commands.scala rename to cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala index 25ec6c882..c74da9917 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Commands.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala @@ -2,228 +2,17 @@ package coursier package cli import java.io.{ FileInputStream, ByteArrayOutputStream, File, IOException } -import java.net.URLClassLoader -import java.nio.file.{ Files => NIOFiles } +import java.nio.file.Files import java.nio.file.attribute.PosixFilePermission import java.util.Properties import java.util.zip.{ ZipEntry, ZipOutputStream, ZipInputStream } -import caseapp.{ HelpMessage => Help, ValueDescription => Value, ExtraName => Short, _ } - -import scala.annotation.tailrec -import scala.language.reflectiveCalls -import scala.util.Try - - -sealed abstract class CoursierCommand extends Command - -case class Resolve( - @Recurse - common: CommonOptions -) extends CoursierCommand { - - // the `val helper = ` part is needed because of DelayedInit it seems - val helper = new Helper(common, remainingArgs, printResultStdout = true) - -} - -case class Fetch( - @Recurse - options: FetchOptions -) extends CoursierCommand { - - val helper = new Helper(options.common, remainingArgs, ignoreErrors = options.force) - - val files0 = helper.fetch(sources = options.sources, javadoc = options.javadoc) - - val out = - if (options.classpath) - files0 - .map(_.toString) - .mkString(File.pathSeparator) - else - files0 - .map(_.toString) - .mkString("\n") - - println(out) - -} - -object Launch { - - @tailrec - def mainClassLoader(cl: ClassLoader): Option[ClassLoader] = - if (cl == null) - None - else { - val isMainLoader = try { - val cl0 = cl.asInstanceOf[Object { - def isBootstrapLoader: Boolean - }] - - cl0.isBootstrapLoader - } catch { - case e: Exception => - false - } - - if (isMainLoader) - Some(cl) - else - mainClassLoader(cl.getParent) - } - -} - -case class Launch( - @Recurse - options: LaunchOptions -) extends CoursierCommand { - - val (rawDependencies, extraArgs) = { - val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) - idxOpt.fold((remainingArgs, Seq.empty[String])) { idx => - val (l, r) = remainingArgs.splitAt(idx) - assert(r.nonEmpty) - (l, r.tail) - } - } - - val helper = new Helper( - options.common, - rawDependencies ++ options.isolated.rawIsolated.map { case (_, dep) => dep } - ) - - - val files0 = helper.fetch(sources = false, javadoc = false) - - val contextLoader = Thread.currentThread().getContextClassLoader - - val parentLoader0: ClassLoader = - if (Try(contextLoader.loadClass("coursier.cli.Launch")).isSuccess) - Launch.mainClassLoader(contextLoader) - .flatMap(cl => Option(cl.getParent)) - .getOrElse { - if (options.common.verbose0 >= 0) - Console.err.println( - "Warning: cannot find the main ClassLoader that launched coursier. " + - "Was coursier launched by its main launcher? " + - "The ClassLoader of the application that is about to be launched will be intertwined " + - "with the one of coursier, which may be a problem if their dependencies conflict." - ) - contextLoader - } - else - // proguarded -> no risk of conflicts, no need to find a specific ClassLoader - contextLoader - - val (parentLoader, filteredFiles) = - if (options.isolated.isolated.isEmpty) - (parentLoader0, files0) - else { - val (isolatedLoader, filteredFiles0) = options.isolated.targets.foldLeft((parentLoader0, files0)) { - case ((parent, files0), target) => - - // FIXME These were already fetched above - val isolatedFiles = helper.fetch( - sources = false, - javadoc = false, - subset = options.isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet - ) - - if (options.common.verbose0 >= 1) { - Console.err.println(s"Isolated loader files:") - for (f <- isolatedFiles.map(_.toString).sorted) - Console.err.println(s" $f") - } - - val isolatedLoader = new IsolatedClassLoader( - isolatedFiles.map(_.toURI.toURL).toArray, - parent, - Array(target) - ) - - val filteredFiles0 = files0.filterNot(isolatedFiles.toSet) - - (isolatedLoader, filteredFiles0) - } - - if (options.common.verbose0 >= 1) { - Console.err.println(s"Remaining files:") - for (f <- filteredFiles0.map(_.toString).sorted) - Console.err.println(s" $f") - } - - (isolatedLoader, filteredFiles0) - } - - val loader = new URLClassLoader( - filteredFiles.map(_.toURI.toURL).toArray, - parentLoader - ) - - val mainClass0 = - if (options.mainClass.nonEmpty) options.mainClass - else { - val mainClasses = Helper.mainClasses(loader) - - val mainClass = - if (mainClasses.isEmpty) { - Helper.errPrintln("No main class found. Specify one with -M or --main.") - sys.exit(255) - } else if (mainClasses.size == 1) { - val (_, mainClass) = mainClasses.head - mainClass - } else { - // Trying to get the main class of the first artifact - val mainClassOpt = for { - (module, _, _) <- helper.moduleVersionConfigs.headOption - mainClass <- mainClasses.collectFirst { - case ((org, name), mainClass) - if org == module.organization && ( - module.name == name || - module.name.startsWith(name + "_") // Ignore cross version suffix - ) => - mainClass - } - } yield mainClass - - mainClassOpt.getOrElse { - Helper.errPrintln(s"Cannot find default main class. Specify one with -M or --main.") - sys.exit(255) - } - } - - mainClass - } - - val cls = - try loader.loadClass(mainClass0) - catch { case e: ClassNotFoundException => - Helper.errPrintln(s"Error: class $mainClass0 not found") - sys.exit(255) - } - val method = - try cls.getMethod("main", classOf[Array[String]]) - catch { case e: NoSuchMethodException => - Helper.errPrintln(s"Error: method main not found in $mainClass0") - sys.exit(255) - } - - if (options.common.verbose0 >= 1) - Helper.errPrintln(s"Launching $mainClass0 ${extraArgs.mkString(" ")}") - else if (options.common.verbose0 == 0) - Helper.errPrintln(s"Launching") - - Thread.currentThread().setContextClassLoader(loader) - method.invoke(null, extraArgs.toArray) -} +import caseapp._ case class Bootstrap( @Recurse options: BootstrapOptions -) extends CoursierCommand { +) extends App { import scala.collection.JavaConverters._ @@ -402,7 +191,7 @@ case class Bootstrap( "exec java -jar " + options.javaOpt.map(s => "'" + s.replace("'", "\\'") + "'").mkString(" ") + " \"$0\" \"$@\"" ).mkString("", "\n", "\n") - try NIOFiles.write(output0.toPath, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) + try Files.write(output0.toPath, shellPreamble.getBytes("UTF-8") ++ buffer.toByteArray) catch { case e: IOException => Console.err.println(s"Error while writing $output0: ${e.getMessage}") sys.exit(1) @@ -410,7 +199,7 @@ case class Bootstrap( try { - val perms = NIOFiles.getPosixFilePermissions(output0.toPath).asScala.toSet + val perms = Files.getPosixFilePermissions(output0.toPath).asScala.toSet var newPerms = perms if (perms(PosixFilePermission.OWNER_READ)) @@ -421,7 +210,7 @@ case class Bootstrap( newPerms += PosixFilePermission.OTHERS_EXECUTE if (newPerms != perms) - NIOFiles.setPosixFilePermissions( + Files.setPosixFilePermissions( output0.toPath, newPerms.asJava ) diff --git a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala index b9801fba1..a2be793d4 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala @@ -2,8 +2,37 @@ package coursier package cli import caseapp._ +import caseapp.core.{ ArgsApp, CommandsMessages } -object Coursier extends CommandAppOf[CoursierCommand] { +import shapeless.union.Union + +// Temporary, see comment in Coursier below +case class CoursierCommandHelper( + command: CoursierCommandHelper.U +) extends ArgsApp { + def setRemainingArgs(remainingArgs: Seq[String]): Unit = + command.unify.setRemainingArgs(remainingArgs) + def remainingArgs: Seq[String] = + command.unify.remainingArgs + def apply(): Unit = + command.unify.apply() +} + +object CoursierCommandHelper { + type U = Union.`'bootstrap -> Bootstrap, 'fetch -> Fetch, 'launch -> Launch, 'resolve -> Resolve`.T + + implicit val commandParser: CommandParser[CoursierCommandHelper] = + CommandParser[U].map(CoursierCommandHelper(_)) + implicit val commandsMessages: CommandsMessages[CoursierCommandHelper] = + CommandsMessages(CommandsMessages[U].messages) +} + +object Coursier extends CommandAppOf[ + // Temporary using CoursierCommandHelper instead of the union type, until case-app + // supports the latter directly. + // Union.`'bootstrap -> Bootstrap, 'fetch -> Fetch, 'launch -> Launch, 'resolve -> Resolve`.T + CoursierCommandHelper +] { override def appName = "Coursier" override def progName = "coursier" override def appVersion = coursier.util.Properties.version diff --git a/cli/src/main/scala-2.11/coursier/cli/Fetch.scala b/cli/src/main/scala-2.11/coursier/cli/Fetch.scala new file mode 100644 index 000000000..3677313be --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/Fetch.scala @@ -0,0 +1,31 @@ +package coursier +package cli + +import java.io.File + +import caseapp._ + +import scala.language.reflectiveCalls + +case class Fetch( + @Recurse + options: FetchOptions +) extends App { + + val helper = new Helper(options.common, remainingArgs, ignoreErrors = options.force) + + val files0 = helper.fetch(sources = options.sources, javadoc = options.javadoc) + + val out = + if (options.classpath) + files0 + .map(_.toString) + .mkString(File.pathSeparator) + else + files0 + .map(_.toString) + .mkString("\n") + + println(out) + +} diff --git a/cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala b/cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala deleted file mode 100644 index c07184672..000000000 --- a/cli/src/main/scala-2.11/coursier/cli/IsolatedClassLoader.scala +++ /dev/null @@ -1,19 +0,0 @@ -package coursier.cli - -import java.net.{ URL, URLClassLoader } - -class IsolatedClassLoader( - urls: Array[URL], - parent: ClassLoader, - isolationTargets: Array[String] -) extends URLClassLoader(urls, parent) { - - /** - * Applications wanting to access an isolated `ClassLoader` should inspect the hierarchy of - * loaders, and look into each of them for this method, by reflection. Then they should - * call it (still by reflection), and look for an agreed in advance target in it. If it is found, - * then the corresponding `ClassLoader` is the one with isolated dependencies. - */ - def getIsolationTargets: Array[String] = isolationTargets - -} diff --git a/cli/src/main/scala-2.11/coursier/cli/Launch.scala b/cli/src/main/scala-2.11/coursier/cli/Launch.scala new file mode 100644 index 000000000..dbd4844e0 --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/Launch.scala @@ -0,0 +1,196 @@ +package coursier +package cli + +import java.net.{ URL, URLClassLoader } + +import caseapp._ + +import scala.annotation.tailrec +import scala.language.reflectiveCalls +import scala.util.Try + +object Launch { + + @tailrec + def mainClassLoader(cl: ClassLoader): Option[ClassLoader] = + if (cl == null) + None + else { + val isMainLoader = try { + val cl0 = cl.asInstanceOf[Object { + def isBootstrapLoader: Boolean + }] + + cl0.isBootstrapLoader + } catch { + case e: Exception => + false + } + + if (isMainLoader) + Some(cl) + else + mainClassLoader(cl.getParent) + } + + class IsolatedClassLoader( + urls: Array[URL], + parent: ClassLoader, + isolationTargets: Array[String] + ) extends URLClassLoader(urls, parent) { + + /** + * Applications wanting to access an isolated `ClassLoader` should inspect the hierarchy of + * loaders, and look into each of them for this method, by reflection. Then they should + * call it (still by reflection), and look for an agreed in advance target in it. If it is found, + * then the corresponding `ClassLoader` is the one with isolated dependencies. + */ + def getIsolationTargets: Array[String] = isolationTargets + + } + +} + +case class Launch( + @Recurse + options: LaunchOptions +) extends App { + + val (rawDependencies, extraArgs) = { + val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) + idxOpt.fold((remainingArgs, Seq.empty[String])) { idx => + val (l, r) = remainingArgs.splitAt(idx) + assert(r.nonEmpty) + (l, r.tail) + } + } + + val helper = new Helper( + options.common, + rawDependencies ++ options.isolated.rawIsolated.map { case (_, dep) => dep } + ) + + + val files0 = helper.fetch(sources = false, javadoc = false) + + val contextLoader = Thread.currentThread().getContextClassLoader + + val parentLoader0: ClassLoader = + if (Try(contextLoader.loadClass("coursier.cli.Launch")).isSuccess) + Launch.mainClassLoader(contextLoader) + .flatMap(cl => Option(cl.getParent)) + .getOrElse { + if (options.common.verbose0 >= 0) + Console.err.println( + "Warning: cannot find the main ClassLoader that launched coursier. " + + "Was coursier launched by its main launcher? " + + "The ClassLoader of the application that is about to be launched will be intertwined " + + "with the one of coursier, which may be a problem if their dependencies conflict." + ) + contextLoader + } + else + // proguarded -> no risk of conflicts, no need to find a specific ClassLoader + contextLoader + + val (parentLoader, filteredFiles) = + if (options.isolated.isolated.isEmpty) + (parentLoader0, files0) + else { + val (isolatedLoader, filteredFiles0) = options.isolated.targets.foldLeft((parentLoader0, files0)) { + case ((parent, files0), target) => + + // FIXME These were already fetched above + val isolatedFiles = helper.fetch( + sources = false, + javadoc = false, + subset = options.isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet + ) + + if (options.common.verbose0 >= 1) { + Console.err.println(s"Isolated loader files:") + for (f <- isolatedFiles.map(_.toString).sorted) + Console.err.println(s" $f") + } + + val isolatedLoader = new Launch.IsolatedClassLoader( + isolatedFiles.map(_.toURI.toURL).toArray, + parent, + Array(target) + ) + + val filteredFiles0 = files0.filterNot(isolatedFiles.toSet) + + (isolatedLoader, filteredFiles0) + } + + if (options.common.verbose0 >= 1) { + Console.err.println(s"Remaining files:") + for (f <- filteredFiles0.map(_.toString).sorted) + Console.err.println(s" $f") + } + + (isolatedLoader, filteredFiles0) + } + + val loader = new URLClassLoader( + filteredFiles.map(_.toURI.toURL).toArray, + parentLoader + ) + + val mainClass0 = + if (options.mainClass.nonEmpty) options.mainClass + else { + val mainClasses = Helper.mainClasses(loader) + + val mainClass = + if (mainClasses.isEmpty) { + Helper.errPrintln("No main class found. Specify one with -M or --main.") + sys.exit(255) + } else if (mainClasses.size == 1) { + val (_, mainClass) = mainClasses.head + mainClass + } else { + // Trying to get the main class of the first artifact + val mainClassOpt = for { + (module, _, _) <- helper.moduleVersionConfigs.headOption + mainClass <- mainClasses.collectFirst { + case ((org, name), mainClass) + if org == module.organization && ( + module.name == name || + module.name.startsWith(name + "_") // Ignore cross version suffix + ) => + mainClass + } + } yield mainClass + + mainClassOpt.getOrElse { + Helper.errPrintln(s"Cannot find default main class. Specify one with -M or --main.") + sys.exit(255) + } + } + + mainClass + } + + val cls = + try loader.loadClass(mainClass0) + catch { case e: ClassNotFoundException => + Helper.errPrintln(s"Error: class $mainClass0 not found") + sys.exit(255) + } + val method = + try cls.getMethod("main", classOf[Array[String]]) + catch { case e: NoSuchMethodException => + Helper.errPrintln(s"Error: method main not found in $mainClass0") + sys.exit(255) + } + + if (options.common.verbose0 >= 1) + Helper.errPrintln(s"Launching $mainClass0 ${extraArgs.mkString(" ")}") + else if (options.common.verbose0 == 0) + Helper.errPrintln(s"Launching") + + Thread.currentThread().setContextClassLoader(loader) + method.invoke(null, extraArgs.toArray) +} \ No newline at end of file diff --git a/cli/src/main/scala-2.11/coursier/cli/Resolve.scala b/cli/src/main/scala-2.11/coursier/cli/Resolve.scala new file mode 100644 index 000000000..fd1a78e86 --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/Resolve.scala @@ -0,0 +1,14 @@ +package coursier +package cli + +import caseapp._ + +case class Resolve( + @Recurse + common: CommonOptions +) extends App { + + // the `val helper = ` part is needed because of DelayedInit it seems + val helper = new Helper(common, remainingArgs, printResultStdout = true) + +} From 265977cdfef294e4a72508f9ccd1bd44bfa8ae41 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:25 +0100 Subject: [PATCH 09/20] Fix verbose option Fixes https://github.com/alexarchambault/coursier/issues/86 --- cli/src/main/scala-2.11/coursier/cli/Options.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Options.scala b/cli/src/main/scala-2.11/coursier/cli/Options.scala index b9238010b..43a00d86d 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Options.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Options.scala @@ -17,7 +17,7 @@ case class CommonOptions( quiet: Boolean, @Help("Increase verbosity (specify several times to increase more)") @Short("v") - verbose: List[Unit], + verbose: Int @@ Counter, @Help("Maximum number of resolution iterations (specify a negative value for unlimited, default: 100)") @Short("N") maxIterations: Int = 100, @@ -57,7 +57,7 @@ case class CommonOptions( @Recurse cacheOptions: CacheOptions ) { - val verbose0 = verbose.length - (if (quiet) 1 else 0) + val verbose0 = Tag.unwrap(verbose) - (if (quiet) 1 else 0) lazy val classifier0 = classifier.flatMap(_.split(',')).filter(_.nonEmpty) } From 60f421346ff36e2c354d1f281497b5dd99b2ce1a Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:26 +0100 Subject: [PATCH 10/20] Adjust default verbosity settings of CLI tool --- .../main/scala-2.11/coursier/cli/Helper.scala | 21 ++++++++++--------- .../main/scala-2.11/coursier/cli/Launch.scala | 12 +++++------ .../scala-2.11/coursier/cli/Options.scala | 2 +- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 7f9ddf768..e4f183c45 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -190,7 +190,7 @@ class Helper( ) val logger = - if (verbose0 >= 0) + if (verbosityLevel >= 0) Some(new TermDisplay(new OutputStreamWriter(System.err))) else None @@ -204,17 +204,17 @@ class Helper( fetchs.tail: _* ) val fetch0 = - if (verbose0 <= 0) fetchQuiet - else { + if (verbosityLevel >= 2) { modVers: Seq[(Module, String)] => val print = Task { errPrintln(s"Getting ${modVers.length} project definition(s)") } print.flatMap(_ => fetchQuiet(modVers)) - } + } else + fetchQuiet - if (verbose0 >= 0) { + if (verbosityLevel >= 1) { errPrintln(s" Dependencies:\n${Print.dependenciesUnknownConfigs(dependencies, Map.empty)}") if (forceVersions.nonEmpty) { @@ -237,8 +237,9 @@ class Helper( lazy val projCache = res.projectCache.mapValues { case (_, p) => p } - if (printResultStdout || verbose0 >= 0) { - errPrintln(s" Result:") + if (printResultStdout || verbosityLevel >= 1) { + if ((printResultStdout && verbosityLevel >= 1) || verbosityLevel >= 2) + errPrintln(s" Result:") val depsStr = Print.dependenciesUnknownConfigs(trDeps, projCache) if (printResultStdout) println(depsStr) @@ -285,7 +286,7 @@ class Helper( subset: Set[Dependency] = null ): Seq[Artifact] = { - if (subset == null && verbose0 >= 0) { + if (subset == null && verbosityLevel >= 1) { val msg = cachePolicies match { case Seq(CachePolicy.LocalOnly) => " Checking artifacts" @@ -319,12 +320,12 @@ class Helper( val artifacts0 = artifacts(sources, javadoc, subset) val logger = - if (verbose0 >= 0) + if (verbosityLevel >= 0) Some(new TermDisplay(new OutputStreamWriter(System.err))) else None - if (verbose0 >= 1 && artifacts0.nonEmpty) + if (verbosityLevel >= 1 && artifacts0.nonEmpty) println(s" Found ${artifacts0.length} artifacts") val tasks = artifacts0.map(artifact => diff --git a/cli/src/main/scala-2.11/coursier/cli/Launch.scala b/cli/src/main/scala-2.11/coursier/cli/Launch.scala index dbd4844e0..128a8e79c 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Launch.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Launch.scala @@ -80,9 +80,9 @@ case class Launch( Launch.mainClassLoader(contextLoader) .flatMap(cl => Option(cl.getParent)) .getOrElse { - if (options.common.verbose0 >= 0) + if (options.common.verbosityLevel >= 0) Console.err.println( - "Warning: cannot find the main ClassLoader that launched coursier. " + + "Warning: cannot find the main ClassLoader that launched coursier.\n" + "Was coursier launched by its main launcher? " + "The ClassLoader of the application that is about to be launched will be intertwined " + "with the one of coursier, which may be a problem if their dependencies conflict." @@ -107,7 +107,7 @@ case class Launch( subset = options.isolated.isolatedDeps.getOrElse(target, Seq.empty).toSet ) - if (options.common.verbose0 >= 1) { + if (options.common.verbosityLevel >= 2) { Console.err.println(s"Isolated loader files:") for (f <- isolatedFiles.map(_.toString).sorted) Console.err.println(s" $f") @@ -124,7 +124,7 @@ case class Launch( (isolatedLoader, filteredFiles0) } - if (options.common.verbose0 >= 1) { + if (options.common.verbosityLevel >= 2) { Console.err.println(s"Remaining files:") for (f <- filteredFiles0.map(_.toString).sorted) Console.err.println(s" $f") @@ -186,9 +186,9 @@ case class Launch( sys.exit(255) } - if (options.common.verbose0 >= 1) + if (options.common.verbosityLevel >= 2) Helper.errPrintln(s"Launching $mainClass0 ${extraArgs.mkString(" ")}") - else if (options.common.verbose0 == 0) + else if (options.common.verbosityLevel == 1) Helper.errPrintln(s"Launching") Thread.currentThread().setContextClassLoader(loader) diff --git a/cli/src/main/scala-2.11/coursier/cli/Options.scala b/cli/src/main/scala-2.11/coursier/cli/Options.scala index 43a00d86d..1e2a2d887 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Options.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Options.scala @@ -57,7 +57,7 @@ case class CommonOptions( @Recurse cacheOptions: CacheOptions ) { - val verbose0 = Tag.unwrap(verbose) - (if (quiet) 1 else 0) + val verbosityLevel = Tag.unwrap(verbose) - (if (quiet) 1 else 0) lazy val classifier0 = classifier.flatMap(_.split(',')).filter(_.nonEmpty) } From 68dbe4d122fd512c28256575d51cbfea433fbb12 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:26 +0100 Subject: [PATCH 11/20] Set default verbosity level to 0 in SBT plugin --- plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala index 9cf1f2946..dc5f07b30 100644 --- a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala @@ -36,7 +36,7 @@ object CoursierPlugin extends AutoPlugin { coursierChecksums := Seq(Some("SHA-1"), None), coursierArtifactsChecksums := Seq(None), coursierCachePolicy := CachePolicy.FetchMissing, - coursierVerbosity := 1, + coursierVerbosity := 0, coursierResolvers <<= Tasks.coursierResolversTask, coursierSbtResolvers <<= externalResolvers in updateSbtClassifiers, coursierCache := Cache.default, From 5bd4184ac978cb597d3ab5eb82c784ccbeab500c Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:26 +0100 Subject: [PATCH 12/20] Replace Either[Unit, Unit] by a more explicit custom ADT --- .../src/main/scala/coursier/TermDisplay.scala | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index a02924d15..a45fde470 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -124,9 +124,8 @@ class TermDisplay( Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { case None => helper(lineCount) - case Some(Left(())) => // poison pill - case Some(Right(())) => - // update display + case Some(Message.Stop) => // poison pill + case Some(Message.Update) => val (done0, downloads0) = downloads.synchronized { val q = doneQueue @@ -182,8 +181,8 @@ class TermDisplay( @tailrec def fallbackHelper(previous: Set[String]): Unit = Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { case None => fallbackHelper(previous) - case Some(Left(())) => // poison pill - case Some(Right(())) => + case Some(Message.Stop) => // poison pill + case Some(Message.Update) => val downloads0 = downloads.synchronized { downloads .toVector @@ -238,7 +237,7 @@ class TermDisplay( for (_ <- 0 until currentHeight) { out.up(2) } - q.put(Left(())) + q.put(Message.Stop) lock.synchronized(()) } @@ -332,10 +331,16 @@ class TermDisplay( private val doneQueue = new ArrayBuffer[(String, Info)] private val infos = new ConcurrentHashMap[String, Info] - private val q = new LinkedBlockingDeque[Either[Unit, Unit]] + private sealed abstract class Message extends Product with Serializable + private object Message { + case object Update extends Message + case object Stop extends Message + } + + private val q = new LinkedBlockingDeque[Message] def update(): Unit = { if (q.size() == 0) - q.put(Right(())) + q.put(Message.Update) } private def newEntry( From 3cc88c5606800ef668f754d58047177b5722e786 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:26 +0100 Subject: [PATCH 13/20] Adjust verbosity and cache policies settings in SBT plugin - Set default verbosity level to 0 - Use the same cache policies as the CLI app everywhere (local-only, then fetch-missing) - Allow to manually override these via the environment (COURSIER_VERBOSITY and COURSIER_MODE), or Java properties (coursier.verbosity and coursier.mode) --- .../scala-2.10/coursier/CoursierPlugin.scala | 6 +- .../src/main/scala-2.10/coursier/Keys.scala | 2 +- .../main/scala-2.10/coursier/Settings.scala | 81 +++++++++++++++++++ .../src/main/scala-2.10/coursier/Tasks.scala | 41 +++++++--- 4 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 plugin/src/main/scala-2.10/coursier/Settings.scala diff --git a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala index dc5f07b30..01db819d8 100644 --- a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala @@ -16,7 +16,7 @@ object CoursierPlugin extends AutoPlugin { val coursierMaxIterations = Keys.coursierMaxIterations val coursierChecksums = Keys.coursierChecksums val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums - val coursierCachePolicy = Keys.coursierCachePolicy + val coursierCachePolicies = Keys.coursierCachePolicies val coursierVerbosity = Keys.coursierVerbosity val coursierResolvers = Keys.coursierResolvers val coursierSbtResolvers = Keys.coursierSbtResolvers @@ -35,8 +35,8 @@ object CoursierPlugin extends AutoPlugin { coursierMaxIterations := 50, coursierChecksums := Seq(Some("SHA-1"), None), coursierArtifactsChecksums := Seq(None), - coursierCachePolicy := CachePolicy.FetchMissing, - coursierVerbosity := 0, + coursierCachePolicies := Settings.defaultCachePolicies, + coursierVerbosity := Settings.defaultVerbosityLevel, coursierResolvers <<= Tasks.coursierResolversTask, coursierSbtResolvers <<= externalResolvers in updateSbtClassifiers, coursierCache := Cache.default, diff --git a/plugin/src/main/scala-2.10/coursier/Keys.scala b/plugin/src/main/scala-2.10/coursier/Keys.scala index d870f320d..0af9153d9 100644 --- a/plugin/src/main/scala-2.10/coursier/Keys.scala +++ b/plugin/src/main/scala-2.10/coursier/Keys.scala @@ -9,7 +9,7 @@ object Keys { val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations", "") val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "") val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums", "") - val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "") + val coursierCachePolicies = SettingKey[Seq[CachePolicy]]("coursier-cache-policies", "") val coursierVerbosity = SettingKey[Int]("coursier-verbosity", "") diff --git a/plugin/src/main/scala-2.10/coursier/Settings.scala b/plugin/src/main/scala-2.10/coursier/Settings.scala new file mode 100644 index 000000000..2a63999be --- /dev/null +++ b/plugin/src/main/scala-2.10/coursier/Settings.scala @@ -0,0 +1,81 @@ +package coursier + +import scala.util.{Failure, Success, Try} + +object Settings { + + private val baseDefaultVerbosityLevel = 0 + + def defaultVerbosityLevel: Int = { + + def fromOption(value: Option[String], description: String): Option[Int] = + value.filter(_.nonEmpty).flatMap { + str => + Try(str.toInt) match { + case Success(level) => Some(level) + case Failure(ex) => + Console.err.println( + s"Warning: unrecognized $description value (should be an integer), ignoring it." + ) + None + } + } + + val fromEnv = fromOption( + sys.env.get("COURSIER_VERBOSITY"), + "COURSIER_VERBOSITY environment variable" + ) + + def fromProps = fromOption( + sys.props.get("coursier.verbosity"), + "Java property coursier.verbosity" + ) + + fromEnv + .orElse(fromProps) + .getOrElse(baseDefaultVerbosityLevel) + } + + + private val baseDefaultCachePolicies = Seq( + CachePolicy.LocalOnly, + CachePolicy.FetchMissing + ) + + def defaultCachePolicies: Seq[CachePolicy] = { + + def fromOption(value: Option[String], description: String): Option[Seq[CachePolicy]] = + value.filter(_.nonEmpty).flatMap { + str => + CacheParse.cachePolicies(str) match { + case scalaz.Success(Seq()) => + Console.err.println( + s"Warning: no mode found in $description, ignoring it." + ) + None + case scalaz.Success(policies) => + Some(policies) + case scalaz.Failure(errors) => + Console.err.println( + s"Warning: unrecognized mode in $description, ignoring it." + ) + None + } + } + + val fromEnv = fromOption( + sys.env.get("COURSIER_MODE"), + "COURSIER_MODE environment variable" + ) + + def fromProps = fromOption( + sys.props.get("coursier.mode"), + "Java property coursier.mode" + ) + + fromEnv + .orElse(fromProps) + .getOrElse(baseDefaultCachePolicies) + } + +} diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index 7f8d3a425..0d62f6404 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -216,7 +216,7 @@ object Tasks { val checksums = coursierChecksums.value val artifactsChecksums = coursierArtifactsChecksums.value val maxIterations = coursierMaxIterations.value - val cachePolicy = coursierCachePolicy.value + val cachePolicies = coursierCachePolicies.value val cache = coursierCache.value val sv = scalaVersion.value // is this always defined? (e.g. for Java only projects?) @@ -227,7 +227,7 @@ object Tasks { else coursierResolvers.value - val verbosity = coursierVerbosity.value + val verbosityLevel = coursierVerbosity.value val startRes = Resolution( @@ -252,7 +252,7 @@ object Tasks { Files.write(cacheIvyPropertiesFile.toPath, "".getBytes("UTF-8")) } - if (verbosity >= 2) { + if (verbosityLevel >= 2) { println("InterProjectRepository") for (p <- projects) println(s" ${p.module}:${p.version}") @@ -283,8 +283,10 @@ object Tasks { val fetch = Fetch.from( repositories, - Cache.fetch(cache, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool), - Cache.fetch(cache, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool) + Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool), + cachePolicies.tail.map(p => + Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool) + ): _* ) def depsRepr(deps: Seq[(String, Dependency)]) = @@ -297,7 +299,7 @@ object Tasks { s"${dep.module}:${dep.version}:${dep.configuration}" }.sorted.distinct - if (verbosity >= 1) { + if (verbosityLevel >= 1) { val repoReprs = repositories.map { case r: IvyRepository => s"ivy:${r.pattern}" @@ -313,9 +315,9 @@ object Tasks { errPrintln(s"Repositories:\n${repoReprs.map(" "+_).mkString("\n")}") } - if (verbosity >= 0) + if (verbosityLevel >= 0) errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") - if (verbosity >= 1) + if (verbosityLevel >= 1) for (depRepr <- depsRepr(currentProject.dependencies)) errPrintln(s" $depRepr") @@ -374,9 +376,9 @@ object Tasks { } } - if (verbosity >= 0) + if (verbosityLevel >= 0) errPrintln("Resolution done") - if (verbosity >= 1) { + if (verbosityLevel >= 1) { val finalDeps = Config.dependenciesWithConfig( res, depsByConfig.map { case (k, l) => k -> l.toSet }, @@ -408,10 +410,23 @@ object Tasks { val artifactsLogger = createLogger() val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => - Cache.file(a, cache, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _)) + def f(p: CachePolicy) = + Cache.file( + a, + cache, + p, + checksums = artifactsChecksums, + logger = Some(artifactsLogger), + pool = pool + ) + + cachePolicies.tail + .foldLeft(f(cachePolicies.head))(_ orElse f(_)) + .run + .map((a, _)) } - if (verbosity >= 0) + if (verbosityLevel >= 0) errPrintln(s"Fetching artifacts") artifactsLogger.init() @@ -425,7 +440,7 @@ object Tasks { artifactsLogger.stop() - if (verbosity >= 0) + if (verbosityLevel >= 0) errPrintln(s"Fetching artifacts: done") def artifactFileOpt(artifact: Artifact) = { From 80f94a360bffd9c856db79b756b7189a20ad1f5d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:27 +0100 Subject: [PATCH 14/20] Disable progress bars by default in non-interactive mode --- cache/src/main/scala/coursier/TermDisplay.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index a45fde470..efa423f47 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -50,9 +50,18 @@ object Terminal { } +object TermDisplay { + private def defaultFallbackMode: Boolean = { + val env = sys.env.get("COURSIER_NO_TERM").nonEmpty + def nonInteractive = System.console() == null + + env || nonInteractive + } +} + class TermDisplay( out: Writer, - var fallbackMode: Boolean = sys.env.get("COURSIER_NO_TERM").nonEmpty + var fallbackMode: Boolean = TermDisplay.defaultFallbackMode ) extends Cache.Logger { import Terminal.Ansi From 3ff533a02be82813df2f03e65932f4b03471fa5e Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:27 +0100 Subject: [PATCH 15/20] Move things around in TermDisplay --- build.sbt | 3 + .../src/main/scala/coursier/TermDisplay.scala | 401 +++++++++--------- 2 files changed, 213 insertions(+), 191 deletions(-) diff --git a/build.sbt b/build.sbt index 06f6dbe38..b2558732f 100644 --- a/build.sbt +++ b/build.sbt @@ -221,6 +221,9 @@ lazy val cache = project Seq( // Since 1.0.0-M10 + // methods that should have been private anyway + ProblemFilters.exclude[MissingMethodProblem]("coursier.TermDisplay.update"), + ProblemFilters.exclude[MissingMethodProblem]("coursier.TermDisplay.fallbackMode_="), // cache argument type changed from `Seq[(String, File)]` to `File` ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.file"), ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.fetch"), diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index efa423f47..3c6fafc10 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -57,23 +57,189 @@ object TermDisplay { env || nonInteractive } -} -class TermDisplay( - out: Writer, - var fallbackMode: Boolean = TermDisplay.defaultFallbackMode -) extends Cache.Logger { - import Terminal.Ansi + private sealed abstract class Info extends Product with Serializable { + def fraction: Option[Double] + def display(): String + } + + private case class DownloadInfo( + downloaded: Long, + previouslyDownloaded: Long, + length: Option[Long], + startTime: Long, + updateCheck: Boolean + ) extends Info { + /** 0.0 to 1.0 */ + def fraction: Option[Double] = length.map(downloaded.toDouble / _) + /** Byte / s */ + def rate(): Option[Double] = { + val currentTime = System.currentTimeMillis() + if (currentTime > startTime) + Some((downloaded - previouslyDownloaded).toDouble / (System.currentTimeMillis() - startTime) * 1000.0) + else + None + } + + // Scala version of http://stackoverflow.com/questions/3758606/how-to-convert-byte-size-into-human-readable-format-in-java/3758880#3758880 + private def byteCount(bytes: Long, si: Boolean = false) = { + val unit = if (si) 1000 else 1024 + if (bytes < unit) + bytes + " B" + else { + val exp = (math.log(bytes) / math.log(unit)).toInt + val pre = (if (si) "kMGTPE" else "KMGTPE").charAt(exp - 1) + (if (si) "" else "i") + f"${bytes / math.pow(unit, exp)}%.1f ${pre}B" + } + } + + def display(): String = { + val decile = (10.0 * fraction.getOrElse(0.0)).toInt + assert(decile >= 0) + assert(decile <= 10) + + fraction.fold(" " * 6)(p => f"${100.0 * p}%5.1f%%") + + " [" + ("#" * decile) + (" " * (10 - decile)) + "] " + + byteCount(downloaded) + + rate().fold("")(r => s" (${byteCount(r.toLong)} / s)") + } + } + + private val format = + new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + private def formatTimestamp(ts: Long): String = + format.format(new Timestamp(ts)) + + private case class CheckUpdateInfo( + currentTimeOpt: Option[Long], + remoteTimeOpt: Option[Long], + isDone: Boolean + ) extends Info { + def fraction = None + def display(): String = { + if (isDone) + (currentTimeOpt, remoteTimeOpt) match { + case (Some(current), Some(remote)) => + if (current < remote) + s"Updated since ${formatTimestamp(current)} (${formatTimestamp(remote)})" + else if (current == remote) + s"No new update since ${formatTimestamp(current)}" + else + s"Warning: local copy newer than remote one (${formatTimestamp(current)} > ${formatTimestamp(remote)})" + case (Some(_), None) => + // FIXME Likely a 404 Not found, that should be taken into account by the cache + "No modified time in response" + case (None, Some(remote)) => + s"Last update: ${formatTimestamp(remote)}" + case (None, None) => + "" // ??? + } + else + currentTimeOpt match { + case Some(current) => + s"Checking for updates since ${formatTimestamp(current)}" + case None => + "" // ??? + } + } + } + + private sealed abstract class Message extends Product with Serializable + private object Message { + case object Update extends Message + case object Stop extends Message + } - private var width = 80 private val refreshInterval = 1000 / 60 private val fallbackRefreshInterval = 1000 - private val lock = new AnyRef - private var currentHeight = 0 - private val t = new Thread("TermDisplay") { - override def run() = lock.synchronized { + private class UpdateDisplayThread( + out: Writer, + var fallbackMode: Boolean + ) extends Thread("TermDisplay") { + + import Terminal.Ansi + + setDaemon(true) + + private var width = 80 + private var currentHeight = 0 + + private val q = new LinkedBlockingDeque[Message] + + + def update(): Unit = { + if (q.size() == 0) + q.put(Message.Update) + } + + def end(): Unit = { + q.put(Message.Stop) + join() + } + + private val downloads = new ArrayBuffer[String] + private val doneQueue = new ArrayBuffer[(String, Info)] + val infos = new ConcurrentHashMap[String, Info] + + def newEntry( + url: String, + info: Info, + fallbackMessage: => String + ): Unit = { + assert(!infos.containsKey(url)) + val prev = infos.putIfAbsent(url, info) + assert(prev == null) + + if (fallbackMode) { + // FIXME What about concurrent accesses to out from the thread above? + out.write(fallbackMessage) + out.flush() + } + + downloads.synchronized { + downloads.append(url) + } + + update() + } + + def removeEntry( + url: String, + success: Boolean, + fallbackMessage: => String + )( + update0: Info => Info + ): Unit = { + downloads.synchronized { + downloads -= url + + val info = infos.remove(url) + assert(info != null) + + if (success) + doneQueue += (url -> update0(info)) + } + + if (fallbackMode && success) { + // FIXME What about concurrent accesses to out from the thread above? + out.write(fallbackMessage) + out.flush() + } + + update() + } + + override def run(): Unit = { + + Terminal.consoleDim("cols") match { + case Some(cols) => + width = cols + out.clearLine(2) + case None => + fallbackMode = true + } val baseExtraWidth = width / 5 @@ -191,6 +357,16 @@ class TermDisplay( Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { case None => fallbackHelper(previous) case Some(Message.Stop) => // poison pill + + // clean up display + for (_ <- 1 to 2; _ <- 0 until currentHeight) { + out.clearLine(2) + out.down(1) + } + for (_ <- 0 until currentHeight) { + out.up(2) + } + case Some(Message.Update) => val downloads0 = downloads.synchronized { downloads @@ -215,7 +391,7 @@ class TermDisplay( out.flush() Thread.sleep(fallbackRefreshInterval) fallbackHelper(previous ++ downloads0.map { case (url, _) => url }) - } + } if (fallbackMode) fallbackHelper(Set.empty) @@ -224,191 +400,34 @@ class TermDisplay( } } - t.setDaemon(true) +} + +class TermDisplay( + out: Writer, + val fallbackMode: Boolean = TermDisplay.defaultFallbackMode +) extends Cache.Logger { + + import TermDisplay._ + + private val updateThread = new UpdateDisplayThread(out, fallbackMode) def init(): Unit = { - Terminal.consoleDim("cols") match { - case Some(cols) => - width = cols - out.clearLine(2) - case None => - fallbackMode = true - } - - t.start() + updateThread.start() } def stop(): Unit = { - for (_ <- 1 to 2; _ <- 0 until currentHeight) { - out.clearLine(2) - out.down(1) - } - for (_ <- 0 until currentHeight) { - out.up(2) - } - q.put(Message.Stop) - lock.synchronized(()) - } - - private sealed abstract class Info extends Product with Serializable { - def fraction: Option[Double] - def display(): String - } - - private case class DownloadInfo( - downloaded: Long, - previouslyDownloaded: Long, - length: Option[Long], - startTime: Long, - updateCheck: Boolean - ) extends Info { - /** 0.0 to 1.0 */ - def fraction: Option[Double] = length.map(downloaded.toDouble / _) - /** Byte / s */ - def rate(): Option[Double] = { - val currentTime = System.currentTimeMillis() - if (currentTime > startTime) - Some((downloaded - previouslyDownloaded).toDouble / (System.currentTimeMillis() - startTime) * 1000.0) - else - None - } - - // Scala version of http://stackoverflow.com/questions/3758606/how-to-convert-byte-size-into-human-readable-format-in-java/3758880#3758880 - private def byteCount(bytes: Long, si: Boolean = false) = { - val unit = if (si) 1000 else 1024 - if (bytes < unit) - bytes + " B" - else { - val exp = (math.log(bytes) / math.log(unit)).toInt - val pre = (if (si) "kMGTPE" else "KMGTPE").charAt(exp - 1) + (if (si) "" else "i") - f"${bytes / math.pow(unit, exp)}%.1f ${pre}B" - } - } - - def display(): String = { - val decile = (10.0 * fraction.getOrElse(0.0)).toInt - assert(decile >= 0) - assert(decile <= 10) - - fraction.fold(" " * 6)(p => f"${100.0 * p}%5.1f%%") + - " [" + ("#" * decile) + (" " * (10 - decile)) + "] " + - byteCount(downloaded) + - rate().fold("")(r => s" (${byteCount(r.toLong)} / s)") - } - } - - private val format = - new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss") - private def formatTimestamp(ts: Long): String = - format.format(new Timestamp(ts)) - - private case class CheckUpdateInfo( - currentTimeOpt: Option[Long], - remoteTimeOpt: Option[Long], - isDone: Boolean - ) extends Info { - def fraction = None - def display(): String = { - if (isDone) - (currentTimeOpt, remoteTimeOpt) match { - case (Some(current), Some(remote)) => - if (current < remote) - s"Updated since ${formatTimestamp(current)} (${formatTimestamp(remote)})" - else if (current == remote) - s"No new update since ${formatTimestamp(current)}" - else - s"Warning: local copy newer than remote one (${formatTimestamp(current)} > ${formatTimestamp(remote)})" - case (Some(_), None) => - // FIXME Likely a 404 Not found, that should be taken into account by the cache - "No modified time in response" - case (None, Some(remote)) => - s"Last update: ${formatTimestamp(remote)}" - case (None, None) => - "" // ??? - } - else - currentTimeOpt match { - case Some(current) => - s"Checking for updates since ${formatTimestamp(current)}" - case None => - "" // ??? - } - } - } - - private val downloads = new ArrayBuffer[String] - private val doneQueue = new ArrayBuffer[(String, Info)] - private val infos = new ConcurrentHashMap[String, Info] - - private sealed abstract class Message extends Product with Serializable - private object Message { - case object Update extends Message - case object Stop extends Message - } - - private val q = new LinkedBlockingDeque[Message] - def update(): Unit = { - if (q.size() == 0) - q.put(Message.Update) - } - - private def newEntry( - url: String, - info: Info, - fallbackMessage: => String - ): Unit = { - assert(!infos.containsKey(url)) - val prev = infos.putIfAbsent(url, info) - assert(prev == null) - - if (fallbackMode) { - // FIXME What about concurrent accesses to out from the thread above? - out.write(fallbackMessage) - out.flush() - } - - downloads.synchronized { - downloads.append(url) - } - - update() - } - - private def removeEntry( - url: String, - success: Boolean, - fallbackMessage: => String - )( - update0: Info => Info - ): Unit = { - downloads.synchronized { - downloads -= url - - val info = infos.remove(url) - assert(info != null) - - if (success) - doneQueue += (url -> update0(info)) - } - - if (fallbackMode && success) { - // FIXME What about concurrent accesses to out from the thread above? - out.write(fallbackMessage) - out.flush() - } - - update() + updateThread.end() } override def downloadingArtifact(url: String, file: File): Unit = - newEntry( + updateThread.newEntry( url, DownloadInfo(0L, 0L, None, System.currentTimeMillis(), updateCheck = false), s"Downloading $url\n" ) override def downloadLength(url: String, totalLength: Long, alreadyDownloaded: Long): Unit = { - val info = infos.get(url) + val info = updateThread.infos.get(url) assert(info != null) val newInfo = info match { case info0: DownloadInfo => @@ -416,12 +435,12 @@ class TermDisplay( case _ => throw new Exception(s"Incoherent display state for $url") } - infos.put(url, newInfo) + updateThread.infos.put(url, newInfo) - update() + updateThread.update() } override def downloadProgress(url: String, downloaded: Long): Unit = { - val info = infos.get(url) + val info = updateThread.infos.get(url) assert(info != null) val newInfo = info match { case info0: DownloadInfo => @@ -429,16 +448,16 @@ class TermDisplay( case _ => throw new Exception(s"Incoherent display state for $url") } - infos.put(url, newInfo) + updateThread.infos.put(url, newInfo) - update() + updateThread.update() } override def downloadedArtifact(url: String, success: Boolean): Unit = - removeEntry(url, success, s"Downloaded $url\n")(x => x) + updateThread.removeEntry(url, success, s"Downloaded $url\n")(x => x) override def checkingUpdates(url: String, currentTimeOpt: Option[Long]): Unit = - newEntry( + updateThread.newEntry( url, CheckUpdateInfo(currentTimeOpt, None, isDone = false), s"Checking $url\n" @@ -457,7 +476,7 @@ class TermDisplay( } } - removeEntry(url, !newUpdate, s"Checked $url") { + updateThread.removeEntry(url, !newUpdate, s"Checked $url") { case info: CheckUpdateInfo => info.copy(remoteTimeOpt = remoteTimeOpt, isDone = true) case _ => From 074e806c3be20f9756595615a5e8643f57b6f659 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:27 +0100 Subject: [PATCH 16/20] Keep moving things around in TermDisplay --- .../src/main/scala/coursier/TermDisplay.scala | 307 +++++++++--------- 1 file changed, 153 insertions(+), 154 deletions(-) diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala index 3c6fafc10..25284e081 100644 --- a/cache/src/main/scala/coursier/TermDisplay.scala +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -231,6 +231,157 @@ object TermDisplay { update() } + private def reflowed(url: String, info: Info) = { + val extra = info match { + case downloadInfo: DownloadInfo => + val pctOpt = downloadInfo.fraction.map(100.0 * _) + + if (downloadInfo.length.isEmpty && downloadInfo.downloaded == 0L) + "" + else + s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${downloadInfo.downloaded}${downloadInfo.length.map(" / " + _).mkString})" + + case updateInfo: CheckUpdateInfo => + "Checking for updates" + } + + val baseExtraWidth = width / 5 + + val total = url.length + 1 + extra.length + val (url0, extra0) = + if (total >= width) { // or > ? If equal, does it go down 2 lines? + val overflow = total - width + 1 + + val extra0 = + if (extra.length > baseExtraWidth) + extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…" + else + extra + + val total0 = url.length + 1 + extra0.length + val overflow0 = total0 - width + 1 + + val url0 = + if (total0 >= width) + url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…" + else + url + + (url0, extra0) + } else + (url, extra) + + (url0, extra0) + } + + private def truncatedPrintln(s: String): Unit = { + + out.clearLine(2) + + if (s.length <= width) + out.write(s + "\n") + else + out.write(s.take(width - 1) + "…\n") + } + + @tailrec private def updateDisplayLoop(lineCount: Int): Unit = { + currentHeight = lineCount + + Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { + case None => updateDisplayLoop(lineCount) + case Some(Message.Stop) => // poison pill + case Some(Message.Update) => + + val (done0, downloads0) = downloads.synchronized { + val q = doneQueue + .toVector + .filter { + case (url, _) => + !url.endsWith(".sha1") && !url.endsWith(".md5") + } + .sortBy { case (url, _) => url } + + doneQueue.clear() + + val dw = downloads + .toVector + .map { url => url -> infos.get(url) } + .sortBy { case (_, info) => - info.fraction.sum } + + (q, dw) + } + + for ((url, info) <- done0 ++ downloads0) { + assert(info != null, s"Incoherent state ($url)") + + truncatedPrintln(url) + out.clearLine(2) + out.write(s" ${info.display()}\n") + } + + val displayedCount = (done0 ++ downloads0).length + + if (displayedCount < lineCount) { + for (_ <- 1 to 2; _ <- displayedCount until lineCount) { + out.clearLine(2) + out.down(1) + } + + for (_ <- displayedCount until lineCount) + out.up(2) + } + + for (_ <- downloads0.indices) + out.up(2) + + out.left(10000) + + out.flush() + Thread.sleep(refreshInterval) + updateDisplayLoop(downloads0.length) + } + } + + @tailrec private def fallbackDisplayLoop(previous: Set[String]): Unit = + Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { + case None => fallbackDisplayLoop(previous) + case Some(Message.Stop) => // poison pill + + // clean up display + for (_ <- 1 to 2; _ <- 0 until currentHeight) { + out.clearLine(2) + out.down(1) + } + for (_ <- 0 until currentHeight) { + out.up(2) + } + + case Some(Message.Update) => + val downloads0 = downloads.synchronized { + downloads + .toVector + .map { url => url -> infos.get(url) } + .sortBy { case (_, info) => - info.fraction.sum } + } + + var displayedSomething = false + for ((url, info) <- downloads0 if previous(url)) { + assert(info != null, s"Incoherent state ($url)") + + val (url0, extra0) = reflowed(url, info) + + displayedSomething = true + out.write(s"$url0 $extra0\n") + } + + if (displayedSomething) + out.write("\n") + + out.flush() + Thread.sleep(fallbackRefreshInterval) + fallbackDisplayLoop(previous ++ downloads0.map { case (url, _) => url }) + } + override def run(): Unit = { Terminal.consoleDim("cols") match { @@ -241,162 +392,10 @@ object TermDisplay { fallbackMode = true } - val baseExtraWidth = width / 5 - - def reflowed(url: String, info: Info) = { - val extra = info match { - case downloadInfo: DownloadInfo => - val pctOpt = downloadInfo.fraction.map(100.0 * _) - - if (downloadInfo.length.isEmpty && downloadInfo.downloaded == 0L) - "" - else - s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${downloadInfo.downloaded}${downloadInfo.length.map(" / " + _).mkString})" - - case updateInfo: CheckUpdateInfo => - "Checking for updates" - } - - val total = url.length + 1 + extra.length - val (url0, extra0) = - if (total >= width) { // or > ? If equal, does it go down 2 lines? - val overflow = total - width + 1 - - val extra0 = - if (extra.length > baseExtraWidth) - extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…" - else - extra - - val total0 = url.length + 1 + extra0.length - val overflow0 = total0 - width + 1 - - val url0 = - if (total0 >= width) - url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…" - else - url - - (url0, extra0) - } else - (url, extra) - - (url0, extra0) - } - - def truncatedPrintln(s: String): Unit = { - - out.clearLine(2) - - if (s.length <= width) - out.write(s + "\n") - else - out.write(s.take(width - 1) + "…\n") - } - - @tailrec def helper(lineCount: Int): Unit = { - currentHeight = lineCount - - Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { - case None => helper(lineCount) - case Some(Message.Stop) => // poison pill - case Some(Message.Update) => - - val (done0, downloads0) = downloads.synchronized { - val q = doneQueue - .toVector - .filter { - case (url, _) => - !url.endsWith(".sha1") && !url.endsWith(".md5") - } - .sortBy { case (url, _) => url } - - doneQueue.clear() - - val dw = downloads - .toVector - .map { url => url -> infos.get(url) } - .sortBy { case (_, info) => - info.fraction.sum } - - (q, dw) - } - - for ((url, info) <- done0 ++ downloads0) { - assert(info != null, s"Incoherent state ($url)") - - truncatedPrintln(url) - out.clearLine(2) - out.write(s" ${info.display()}\n") - } - - val displayedCount = (done0 ++ downloads0).length - - if (displayedCount < lineCount) { - for (_ <- 1 to 2; _ <- displayedCount until lineCount) { - out.clearLine(2) - out.down(1) - } - - for (_ <- displayedCount until lineCount) - out.up(2) - } - - for (_ <- downloads0.indices) - out.up(2) - - out.left(10000) - - out.flush() - Thread.sleep(refreshInterval) - helper(downloads0.length) - } - } - - - @tailrec def fallbackHelper(previous: Set[String]): Unit = - Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { - case None => fallbackHelper(previous) - case Some(Message.Stop) => // poison pill - - // clean up display - for (_ <- 1 to 2; _ <- 0 until currentHeight) { - out.clearLine(2) - out.down(1) - } - for (_ <- 0 until currentHeight) { - out.up(2) - } - - case Some(Message.Update) => - val downloads0 = downloads.synchronized { - downloads - .toVector - .map { url => url -> infos.get(url) } - .sortBy { case (_, info) => - info.fraction.sum } - } - - var displayedSomething = false - for ((url, info) <- downloads0 if previous(url)) { - assert(info != null, s"Incoherent state ($url)") - - val (url0, extra0) = reflowed(url, info) - - displayedSomething = true - out.write(s"$url0 $extra0\n") - } - - if (displayedSomething) - out.write("\n") - - out.flush() - Thread.sleep(fallbackRefreshInterval) - fallbackHelper(previous ++ downloads0.map { case (url, _) => url }) - } - if (fallbackMode) - fallbackHelper(Set.empty) + fallbackDisplayLoop(Set.empty) else - helper(0) + updateDisplayLoop(0) } } From dfc11151b1a0070e4a5fd9eed21cc7757dcfdf71 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:28 +0100 Subject: [PATCH 17/20] Move FileError to a separate file --- cache/src/main/scala/coursier/Cache.scala | 32 ---------------- cache/src/main/scala/coursier/FileError.scala | 38 +++++++++++++++++++ 2 files changed, 38 insertions(+), 32 deletions(-) create mode 100644 cache/src/main/scala/coursier/FileError.scala diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index b17577b7d..53c14f3a5 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -802,35 +802,3 @@ object Cache { } } - -sealed abstract class FileError(val message: String) extends Product with Serializable - -object FileError { - - final case class DownloadError(reason: String) extends FileError(s"Download error: $reason") - - final case class NotFound(file: String, permanent: Option[Boolean] = None) extends FileError(s"Not found: $file") - - final case class ChecksumNotFound( - sumType: String, - file: String - ) extends FileError(s"$sumType checksum not found: $file") - - final case class ChecksumFormatError( - sumType: String, - file: String - ) extends FileError(s"Unrecognized $sumType checksum format in $file") - - final case class WrongChecksum( - sumType: String, - got: String, - expected: String, - file: String, - sumFile: String - ) extends FileError(s"$sumType checksum validation failed: $file") - - sealed abstract class Recoverable(message: String) extends FileError(message) - final case class Locked(file: File) extends Recoverable(s"Locked: $file") - final case class ConcurrentDownload(url: String) extends Recoverable(s"Concurrent download: $url") - -} diff --git a/cache/src/main/scala/coursier/FileError.scala b/cache/src/main/scala/coursier/FileError.scala new file mode 100644 index 000000000..4386a0e98 --- /dev/null +++ b/cache/src/main/scala/coursier/FileError.scala @@ -0,0 +1,38 @@ +package coursier + +import java.io.File + +sealed abstract class FileError(val message: String) extends Product with Serializable + +object FileError { + + final case class DownloadError(reason: String) extends FileError(s"Download error: $reason") + + final case class NotFound( + file: String, + permanent: Option[Boolean] = None + ) extends FileError(s"Not found: $file") + + final case class ChecksumNotFound( + sumType: String, + file: String + ) extends FileError(s"$sumType checksum not found: $file") + + final case class ChecksumFormatError( + sumType: String, + file: String + ) extends FileError(s"Unrecognized $sumType checksum format in $file") + + final case class WrongChecksum( + sumType: String, + got: String, + expected: String, + file: String, + sumFile: String + ) extends FileError(s"$sumType checksum validation failed: $file") + + sealed abstract class Recoverable(message: String) extends FileError(message) + final case class Locked(file: File) extends Recoverable(s"Locked: $file") + final case class ConcurrentDownload(url: String) extends Recoverable(s"Concurrent download: $url") + +} From 83a08d172cffa85e23fefb9fa6c5fb1c9e59eca5 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 13 Mar 2016 22:57:29 +0100 Subject: [PATCH 18/20] Add per-cache file structure lock To be acquired when creating directories or locks in particular, so that these don't hinder each other. --- cache/src/main/scala/coursier/Cache.scala | 64 ++++++++++++++++++++--- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 53c14f3a5..7f4132348 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -107,11 +107,55 @@ object Cache { helper(alreadyDownloaded) } - private def withLockFor[T](file: File)(f: => FileError \/ T): FileError \/ T = { + private val processStructureLocks = new ConcurrentHashMap[File, AnyRef] + + /** + * Should be acquired when doing operations changing the file structure of the cache (creating + * new directories, creating / acquiring locks, ...), so that these don't hinder each other. + * + * Should hopefully address some transient errors seen on the CI of ensime-server. + */ + private def withStructureLock[T](cache: File)(f: => T): T = { + + val intraProcessLock = Option(processStructureLocks.get(cache)).getOrElse { + val lock = new AnyRef + val prev = Option(processStructureLocks.putIfAbsent(cache, lock)) + prev.getOrElse(lock) + } + + intraProcessLock.synchronized { + val lockFile = new File(cache, ".structure.lock") + lockFile.getParentFile.mkdirs() + var out = new FileOutputStream(lockFile) + + try { + var lock: FileLock = null + try { + lock = out.getChannel.lock() + + try f + finally { + lock.release() + lock = null + out.close() + out = null + lockFile.delete() + } + } + finally if (lock != null) lock.release() + } finally if (out != null) out.close() + } + } + + private def withLockFor[T](cache: File, file: File)(f: => FileError \/ T): FileError \/ T = { val lockFile = new File(file.getParentFile, s"${file.getName}.lock") - lockFile.getParentFile.mkdirs() - var out = new FileOutputStream(lockFile) + var out: FileOutputStream = null + + withStructureLock(cache) { + lockFile.getParentFile.mkdirs() + out = new FileOutputStream(lockFile) + } try { var lock: FileLock = null @@ -378,7 +422,7 @@ object Cache { def remote(file: File, url: String): EitherT[Task, FileError, Unit] = EitherT { Task { - withLockFor(file) { + withLockFor(cache, file) { downloading(url, file, logger) { val tmp = temporaryFile(file) @@ -416,14 +460,18 @@ object Cache { val result = try { - tmp.getParentFile.mkdirs() - val out = new FileOutputStream(tmp, partialDownload) + val out = withStructureLock(cache) { + tmp.getParentFile.mkdirs() + new FileOutputStream(tmp, partialDownload) + } try \/-(readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)) finally out.close() } finally in.close() - file.getParentFile.mkdirs() - NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE) + withStructureLock(cache) { + file.getParentFile.mkdirs() + NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE) + } for (lastModified <- Option(conn.getLastModified) if lastModified > 0L) file.setLastModified(lastModified) From 2fbe57da7a57d2f1a0dee0e6c9981ac3bbd4474d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Mon, 14 Mar 2016 16:20:47 +0100 Subject: [PATCH 19/20] Add support for dependencyOverrides in SBT plugin --- .../main/scala-2.10/coursier/FromSbt.scala | 22 +++++++++++++++---- .../src/main/scala-2.10/coursier/Tasks.scala | 7 +++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/plugin/src/main/scala-2.10/coursier/FromSbt.scala b/plugin/src/main/scala-2.10/coursier/FromSbt.scala index 65c24f7b9..0152cd1a3 100644 --- a/plugin/src/main/scala-2.10/coursier/FromSbt.scala +++ b/plugin/src/main/scala-2.10/coursier/FromSbt.scala @@ -34,7 +34,21 @@ object FromSbt { !k.startsWith(SbtPomExtraProperties.POM_INFO_KEY_PREFIX) } - def dependencies( + def moduleVersion( + module: ModuleID, + scalaVersion: String, + scalaBinaryVersion: String + ): (Module, String) = { + + val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) + + val module0 = Module(module.organization, fullName, FromSbt.attributes(module.extraDependencyAttributes)) + val version = module.revision + + (module0, version) + } + + def dependencies( module: ModuleID, scalaVersion: String, scalaBinaryVersion: String @@ -42,11 +56,11 @@ object FromSbt { // TODO Warn about unsupported properties in `module` - val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) + val (module0, version) = moduleVersion(module, scalaVersion, scalaBinaryVersion) val dep = Dependency( - Module(module.organization, fullName, FromSbt.attributes(module.extraDependencyAttributes)), - module.revision, + module0, + version, exclusions = module.exclusions.map { rule => // FIXME Other `rule` fields are ignored here (rule.organization, rule.name) diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index 0d62f6404..fa56b03c5 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -220,6 +220,11 @@ object Tasks { val cache = coursierCache.value val sv = scalaVersion.value // is this always defined? (e.g. for Java only projects?) + val sbv = scalaBinaryVersion.value + + val userForceVersions = dependencyOverrides.value.map( + FromSbt.moduleVersion(_, sv, sbv) + ).toMap val resolvers = if (sbtClassifiers) @@ -233,7 +238,7 @@ object Tasks { val startRes = Resolution( currentProject.dependencies.map { case (_, dep) => dep }.toSet, filter = Some(dep => !dep.optional), - forceVersions = forcedScalaModules(sv) ++ projects.map(_.moduleVersion) + forceVersions = userForceVersions ++ forcedScalaModules(sv) ++ projects.map(_.moduleVersion) ) // required for publish to be fine, later on From 645100ae3c320b762a057e18f51b94c81506010b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Mon, 14 Mar 2016 16:20:48 +0100 Subject: [PATCH 20/20] Try not to use rhino during scala JS tests Why is rhino back? Should be node... --- build.sbt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index b2558732f..8d3a7517b 100644 --- a/build.sbt +++ b/build.sbt @@ -199,7 +199,8 @@ lazy val tests = crossProject ) .jsSettings( postLinkJSEnv := NodeJSEnv().value, - scalaJSStage in Global := FastOptStage + scalaJSStage in Global := FastOptStage, + scalaJSUseRhino in Global := false ) lazy val testsJvm = tests.jvm.dependsOn(cache % "test")