From ee1a6a68f6eaa18059c9b7c1a7ac3886c6a189f0 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:34 +0100 Subject: [PATCH 01/13] Offline option in CLI tool, minor refactoring in MavenRepository --- .../main/scala/coursier/cli/Coursier.scala | 37 ++++++++------- .../{Fetch.scala => MavenRepository.scala} | 18 +++++--- .../test/scala/coursier/test/JsTests.scala | 4 +- .../{Fetch.scala => MavenRepository.scala} | 19 +++++--- .../coursier/test/compatibility/package.scala | 10 +++-- .../main/scala/coursier/core/Repository.scala | 45 ++++++++++++------- project/Coursier.scala | 5 ++- web/src/main/scala/coursier/web/Backend.scala | 14 +++--- 8 files changed, 90 insertions(+), 62 deletions(-) rename core-js/src/main/scala/coursier/core/{Fetch.scala => MavenRepository.scala} (88%) rename core-jvm/src/main/scala/coursier/core/{Fetch.scala => MavenRepository.scala} (85%) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 6349cb792..0162ebd44 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -4,15 +4,18 @@ package cli import java.io.File import caseapp._ -import coursier.core.{Fetch, Parse, Repository}, Repository.CachePolicy +import coursier.core.{MavenRepository, Parse, Repository}, Repository.CachePolicy import scalaz.concurrent.Task -case class Coursier(scope: List[String], - keepOptional: Boolean, - fetch: Boolean, - @ExtraName("v") verbose: List[Unit], - @ExtraName("N") maxIterations: Int = 100) extends App { +case class Coursier( + scope: List[String], + keepOptional: Boolean, + fetch: Boolean, + @ExtraName("c") offline: Boolean, + @ExtraName("v") verbose: List[Unit], + @ExtraName("N") maxIterations: Int = 100 +) extends App { val verbose0 = verbose.length @@ -26,8 +29,8 @@ case class Coursier(scope: List[String], def fileRepr(f: File) = f.toString - val logger: Fetch.Logger with FilesLogger = - new Fetch.Logger with FilesLogger { + val logger: MavenRepository.Logger with FilesLogger = + new MavenRepository.Logger with FilesLogger { def println(s: String) = Console.err.println(s) def downloading(url: String) = @@ -55,17 +58,13 @@ case class Coursier(scope: List[String], } val cachedMavenCentral = Repository.mavenCentral.copy( - fetch = Repository.mavenCentral.fetch.copy( - cache = Some(centralCacheDir), - logger = if (verbose0 <= 1) None else Some(logger) - ) + cache = Some(centralCacheDir), + logger = if (verbose0 <= 1) None else Some(logger) ) val repositories = Seq[Repository]( cachedMavenCentral, Repository.ivy2Local.copy( - fetch = Repository.ivy2Local.fetch.copy( - logger = if (verbose0 <= 1) None else Some(logger) - ) + logger = if (verbose0 <= 1) None else Some(logger) ) ) @@ -149,13 +148,17 @@ case class Coursier(scope: List[String], if (fetch) { println() - val cachePolicy: CachePolicy = CachePolicy.Default + val cachePolicy: CachePolicy = + if (offline) + CachePolicy.LocalOnly + else + CachePolicy.Default val artifacts = res.artifacts val files = new Files( Seq( - cachedMavenCentral.fetch.root -> centralFilesCacheDir + cachedMavenCentral.root -> centralFilesCacheDir ), () => ???, if (verbose0 <= 0) None else Some(logger) diff --git a/core-js/src/main/scala/coursier/core/Fetch.scala b/core-js/src/main/scala/coursier/core/MavenRepository.scala similarity index 88% rename from core-js/src/main/scala/coursier/core/Fetch.scala rename to core-js/src/main/scala/coursier/core/MavenRepository.scala index 62828ef3e..8eaf229d8 100644 --- a/core-js/src/main/scala/coursier/core/Fetch.scala +++ b/core-js/src/main/scala/coursier/core/MavenRepository.scala @@ -12,7 +12,7 @@ import js.Dynamic.{global => g} import scala.scalajs.js.timers._ -object Fetch { +object MavenRepository { def encodeURIComponent(s: String): String = g.encodeURIComponent(s).asInstanceOf[String] @@ -84,18 +84,24 @@ object Fetch { } -case class Fetch(root: String, - logger: Option[Fetch.Logger] = None) { +case class MavenRepository( + root: String, + ivyLike: Boolean = false, + logger: Option[MavenRepository.Logger] = None +) extends BaseMavenRepository(root, ivyLike) { - def apply(artifact: Artifact, - cachePolicy: Repository.CachePolicy): EitherT[Task, String, String] = { + + def fetch( + artifact: Artifact, + cachePolicy: Repository.CachePolicy + ): EitherT[Task, String, String] = { val url0 = root + artifact.url EitherT( Task { implicit ec => Future(logger.foreach(_.fetching(url0))) - .flatMap(_ => Fetch.get(url0)) + .flatMap(_ => MavenRepository.get(url0)) .map{ s => logger.foreach(_.fetched(url0)); \/-(s) } .recover{case e: Exception => logger.foreach(_.other(url0, e.getMessage)) diff --git a/core-js/src/test/scala/coursier/test/JsTests.scala b/core-js/src/test/scala/coursier/test/JsTests.scala index 82533d4e8..d4747b62c 100644 --- a/core-js/src/test/scala/coursier/test/JsTests.scala +++ b/core-js/src/test/scala/coursier/test/JsTests.scala @@ -1,7 +1,7 @@ package coursier package test -import coursier.core.{Fetch, Repository} +import coursier.core.{Repository, MavenRepository} import coursier.test.compatibility._ import utest._ @@ -18,7 +18,7 @@ object JsTests extends TestSuite { } 'get{ - Fetch.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + MavenRepository.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") .map(core.compatibility.xmlParse) .map{ xml => assert(xml.right.toOption.exists(_.label == "project")) diff --git a/core-jvm/src/main/scala/coursier/core/Fetch.scala b/core-jvm/src/main/scala/coursier/core/MavenRepository.scala similarity index 85% rename from core-jvm/src/main/scala/coursier/core/Fetch.scala rename to core-jvm/src/main/scala/coursier/core/MavenRepository.scala index 73d6b509f..f88f0df2f 100644 --- a/core-jvm/src/main/scala/coursier/core/Fetch.scala +++ b/core-jvm/src/main/scala/coursier/core/MavenRepository.scala @@ -8,14 +8,19 @@ import scala.io.Codec import scalaz._, Scalaz._ import scalaz.concurrent.Task -case class Fetch(root: String, - cache: Option[File] = None, - logger: Option[Fetch.Logger] = None) { +case class MavenRepository( + root: String, + cache: Option[File] = None, + ivyLike: Boolean = false, + logger: Option[MavenRepository.Logger] = None +) extends BaseMavenRepository(root, ivyLike) { val isLocal = root.startsWith("file:///") - def apply(artifact: Artifact, - cachePolicy: Repository.CachePolicy): EitherT[Task, String, String] = { + def fetch( + artifact: Artifact, + cachePolicy: Repository.CachePolicy + ): EitherT[Task, String, String] = { def locally(eitherFile: String \/ File) = { Task { @@ -44,7 +49,7 @@ case class Fetch(root: String, val url = new URL(urlStr) def log = Task(logger.foreach(_.downloading(urlStr))) - def get = Fetch.readFully(url.openStream()) + def get = MavenRepository.readFully(url.openStream()) log.flatMap(_ => get) } @@ -70,7 +75,7 @@ case class Fetch(root: String, } -object Fetch { +object MavenRepository { trait Logger { def downloading(url: String): Unit diff --git a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala index 563d745ac..a08aa54e4 100644 --- a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala +++ b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.core.Fetch +import coursier.core.MavenRepository import scala.concurrent.{ExecutionContext, Future} import scalaz.concurrent.Task @@ -14,10 +14,12 @@ package object compatibility { } def textResource(path: String)(implicit ec: ExecutionContext): Future[String] = Future { - def is = getClass.getClassLoader - .getResource(path).openStream() + def is = getClass + .getClassLoader + .getResource(path) + .openStream() - new String(Fetch.readFullySync(is), "UTF-8") + new String(MavenRepository.readFullySync(is), "UTF-8") } } diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index 54a3d1745..45b722315 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -1,6 +1,6 @@ package coursier.core -import coursier.core.Resolution.ModuleVersion +import coursier.core.Repository.CachePolicy import scalaz.{-\/, \/-, \/, EitherT} import scalaz.concurrent.Task @@ -8,19 +8,21 @@ import scalaz.concurrent.Task import coursier.core.compatibility.encodeURIComponent trait Repository { - def find(module: Module, - version: String, - cachePolicy: Repository.CachePolicy = Repository.CachePolicy.Default): EitherT[Task, String, (Artifact.Source, Project)] + def find( + module: Module, + version: String, + cachePolicy: Repository.CachePolicy = Repository.CachePolicy.Default + ): EitherT[Task, String, (Artifact.Source, Project)] } object Repository { - val mavenCentral = MavenRepository(Fetch("https://repo1.maven.org/maven2/")) + val mavenCentral = MavenRepository("https://repo1.maven.org/maven2/") - val sonatypeReleases = MavenRepository(Fetch("https://oss.sonatype.org/content/repositories/releases/")) - val sonatypeSnapshots = MavenRepository(Fetch("https://oss.sonatype.org/content/repositories/snapshots/")) + val sonatypeReleases = MavenRepository("https://oss.sonatype.org/content/repositories/releases/") + val sonatypeSnapshots = MavenRepository("https://oss.sonatype.org/content/repositories/snapshots/") - lazy val ivy2Local = MavenRepository(Fetch("file://" + sys.props("user.home") + "/.ivy2/local/"), ivyLike = true) + lazy val ivy2Local = MavenRepository("file://" + sys.props("user.home") + "/.ivy2/local/", ivyLike = true) /** @@ -34,9 +36,11 @@ object Repository { * version (e.g. version interval). Which version get chosen depends on * the repository implementation. */ - def find(repositories: Seq[Repository], - module: Module, - version: String): EitherT[Task, Seq[String], (Artifact.Source, Project)] = { + def find( + repositories: Seq[Repository], + module: Module, + version: String + ): EitherT[Task, Seq[String], (Artifact.Source, Project)] = { val lookups = repositories.map(repo => repo -> repo.find(module, version).run) val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Artifact.Source, Project)]) { @@ -131,7 +135,7 @@ object Repository { } } -object MavenRepository { +object BaseMavenRepository { def ivyLikePath(org: String, name: String, @@ -154,7 +158,7 @@ object MavenRepository { project: Project): Seq[Artifact] = { def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = - MavenRepository.ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext) + BaseMavenRepository.ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext) val path = if (ivyLike) @@ -206,13 +210,20 @@ object MavenRepository { } -case class MavenRepository(fetch: Fetch, - ivyLike: Boolean = false) extends Repository { +abstract class BaseMavenRepository( + root: String, + ivyLike: Boolean +) extends Repository { + + def fetch( + artifact: Artifact, + cachePolicy: CachePolicy + ): EitherT[Task, String, String] import Repository._ - import MavenRepository._ + import BaseMavenRepository._ - val source = MavenRepository.Source(fetch.root, ivyLike) + val source = BaseMavenRepository.Source(root, ivyLike) def projectArtifact(module: Module, version: String): Artifact = { val path = ( diff --git a/project/Coursier.scala b/project/Coursier.scala index afe29f9da..6c05d4303 100644 --- a/project/Coursier.scala +++ b/project/Coursier.scala @@ -59,7 +59,8 @@ object CoursierBuild extends Build { organization := "com.github.alexarchambault", scalaVersion := "2.11.6", crossScalaVersions := Seq("2.10.5", "2.11.6"), - resolvers += "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases" + resolvers += "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases", + resolvers += Resolver.sonatypeRepo("snapshots") ) ++ publishingSettings private lazy val commonCoreSettings = commonSettings ++ Seq[Setting[_]]( @@ -123,7 +124,7 @@ object CoursierBuild extends Build { .settings( name := "coursier-cli", libraryDependencies ++= Seq( - "com.github.alexarchambault" %% "case-app" % "0.2.2", + "com.github.alexarchambault" %% "case-app" % "0.3.0-SNAPSHOT", "ch.qos.logback" % "logback-classic" % "1.1.3" ) ++ { if (scalaVersion.value startsWith "2.10.") diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index bb9a6cf2c..708ba73c7 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,7 +1,7 @@ package coursier package web -import coursier.core.{Repository, MavenRepository, Fetch} +import coursier.core.{Repository, BaseMavenRepository, MavenRepository} import japgolly.scalajs.react.vdom.{TagMod, Attr} import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope} @@ -111,7 +111,7 @@ class Backend($: BackendScope[Unit, State]) { g.$("#resLogTab a:last").tab("show") $.modState(_.copy(resolving = true, log = Nil)) - val logger: Fetch.Logger = new Fetch.Logger { + val logger: MavenRepository.Logger = new MavenRepository.Logger { def fetched(url: String) = { println(s"<- $url") $.modState(s => s.copy(log = s"<- $url" +: s.log)) @@ -135,7 +135,7 @@ class Backend($: BackendScope[Unit, State]) { res .process - .run(fetch(s.repositories.map(r => r.copy(fetch = r.fetch.copy(logger = Some(logger))))), 100) + .run(fetch(s.repositories.map(r => r.copy(logger = Some(logger)))), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context @@ -246,7 +246,7 @@ object App { )), <.td(Seq[Seq[TagMod]]( res.projectCache.get(dep.moduleVersion) match { - case Some((source: MavenRepository.Source, proj)) if !source.ivyLike => + case Some((source: BaseMavenRepository.Source, proj)) if !source.ivyLike => // FIXME Maven specific, generalize with source.artifacts val version0 = finalVersionOpt getOrElse dep.version val relPath = @@ -401,14 +401,14 @@ object App { def repoItem(repo: MavenRepository) = <.tr( <.td( - <.a(^.href := repo.fetch.root, - repo.fetch.root + <.a(^.href := repo.root, + repo.root ) ) ) val sortedRepos = repos - .sortBy(repo => repo.fetch.root) + .sortBy(repo => repo.root) <.table(^.`class` := "table", <.thead( From edb1c6f2c5a484542021a03e0504ed7456bb8d04 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:36 +0100 Subject: [PATCH 02/13] Formatting / styling --- .../scala/coursier/core/MavenRepository.scala | 8 +- .../coursier/core/compatibility/package.scala | 23 +-- .../scala/scalaz/concurrent/package.scala | 2 +- .../scala/coursier/core/MavenRepository.scala | 2 +- .../main/scala/coursier/core/Repository.scala | 178 ++++++++++-------- 5 files changed, 116 insertions(+), 97 deletions(-) diff --git a/core-js/src/main/scala/coursier/core/MavenRepository.scala b/core-js/src/main/scala/coursier/core/MavenRepository.scala index 8eaf229d8..7215df392 100644 --- a/core-js/src/main/scala/coursier/core/MavenRepository.scala +++ b/core-js/src/main/scala/coursier/core/MavenRepository.scala @@ -1,14 +1,14 @@ package coursier package core -import org.scalajs.dom.raw.{Event, XMLHttpRequest} +import org.scalajs.dom.raw.{ Event, XMLHttpRequest } -import scala.concurrent.{ExecutionContext, Promise, Future} -import scalaz.{-\/, \/-, EitherT} +import scala.concurrent.{ ExecutionContext, Promise, Future } +import scalaz.{ -\/, \/-, EitherT } import scalaz.concurrent.Task import scala.scalajs.js -import js.Dynamic.{global => g} +import js.Dynamic.{ global => g } import scala.scalajs.js.timers._ diff --git a/core-js/src/main/scala/coursier/core/compatibility/package.scala b/core-js/src/main/scala/coursier/core/compatibility/package.scala index 5c940cb9d..34c0f71de 100644 --- a/core-js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core-js/src/main/scala/coursier/core/compatibility/package.scala @@ -1,7 +1,7 @@ package coursier.core import scala.scalajs.js -import js.Dynamic.{global => g} +import js.Dynamic.{ global => g } import org.scalajs.dom.raw.NodeList package object compatibility { @@ -21,25 +21,16 @@ package object compatibility { def letter: Boolean = between(c, 'a', 'z') || between(c, 'A', 'Z') } - lazy val DOMParser = { - import js.Dynamic.{global => g} - import js.DynamicImplicits._ + def newFromXmlDomOrGlobal(name: String) = { + var defn = g.selectDynamic(name) + if (js.isUndefined(defn)) + defn = g.require("xmldom").selectDynamic(name) - val defn = - if (js.isUndefined(g.DOMParser)) g.require("xmldom").DOMParser - else g.DOMParser js.Dynamic.newInstance(defn)() } - lazy val XMLSerializer = { - import js.Dynamic.{global => g} - import js.DynamicImplicits._ - - val defn = - if (js.isUndefined(g.XMLSerializer)) g.require("xmldom").XMLSerializer - else g.XMLSerializer - js.Dynamic.newInstance(defn)() - } + lazy val DOMParser = newFromXmlDomOrGlobal("DOMParser") + lazy val XMLSerializer = newFromXmlDomOrGlobal("XMLSerializer") // Can't find these from node val ELEMENT_NODE = 1 // org.scalajs.dom.raw.Node.ELEMENT_NODE diff --git a/core-js/src/main/scala/scalaz/concurrent/package.scala b/core-js/src/main/scala/scalaz/concurrent/package.scala index 095f06222..0a8a8591f 100644 --- a/core-js/src/main/scala/scalaz/concurrent/package.scala +++ b/core-js/src/main/scala/scalaz/concurrent/package.scala @@ -1,6 +1,6 @@ package scalaz -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.{ ExecutionContext, Future } /** Minimal Future-based Task */ package object concurrent { diff --git a/core-jvm/src/main/scala/coursier/core/MavenRepository.scala b/core-jvm/src/main/scala/coursier/core/MavenRepository.scala index f88f0df2f..c9cc2d97c 100644 --- a/core-jvm/src/main/scala/coursier/core/MavenRepository.scala +++ b/core-jvm/src/main/scala/coursier/core/MavenRepository.scala @@ -2,7 +2,7 @@ package coursier package core import java.io._ -import java.net.{URI, URL} +import java.net.{ URI, URL } import scala.io.Codec import scalaz._, Scalaz._ diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index 45b722315..ef35e01db 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -2,7 +2,7 @@ package coursier.core import coursier.core.Repository.CachePolicy -import scalaz.{-\/, \/-, \/, EitherT} +import scalaz.{ -\/, \/-, \/, EitherT } import scalaz.concurrent.Task import coursier.core.compatibility.encodeURIComponent @@ -42,28 +42,34 @@ object Repository { version: String ): EitherT[Task, Seq[String], (Artifact.Source, Project)] = { - val lookups = repositories.map(repo => repo -> repo.find(module, version).run) - val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Artifact.Source, Project)]) { - case (acc, (repo, t)) => - acc.flatMap { - case -\/(errors) => - t.map(res => res - .flatMap{case (source, project) => - if (project.module == module) \/-((source, project)) - else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") - } - .leftMap(error => error +: errors) - ) + val lookups = repositories + .map(repo => repo -> repo.find(module, version).run) - case res @ \/-(_) => - Task.now(res) - } - } + val task = lookups + .foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Artifact.Source, Project)]) { + case (acc, (repo, eitherProjTask)) => + acc + .flatMap { + case -\/(errors) => + eitherProjTask + .map(res => res + .flatMap{case (source, project) => + if (project.module == module) \/-((source, project)) + else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") + } + .leftMap(error => error +: errors) + ) - EitherT(task.map(_.leftMap(_.reverse))).map {case x @ (_, proj) => - assert(proj.module == module) - x - } + case res @ \/-(_) => + Task.now(res) + } + } + + EitherT(task.map(_.leftMap(_.reverse))) + .map {case x @ (_, proj) => + assert(proj.module == module) + x + } } sealed trait CachePolicy { @@ -137,12 +143,14 @@ object Repository { object BaseMavenRepository { - def ivyLikePath(org: String, - name: String, - version: String, - subDir: String, - baseSuffix: String, - ext: String) = + def ivyLikePath( + org: String, + name: String, + version: String, + subDir: String, + baseSuffix: String, + ext: String + ) = Seq( org, name, @@ -154,11 +162,20 @@ object BaseMavenRepository { case class Source(root: String, ivyLike: Boolean) extends Artifact.Source { import Repository._ - def artifacts(dependency: Dependency, - project: Project): Seq[Artifact] = { + def artifacts( + dependency: Dependency, + project: Project + ): Seq[Artifact] = { def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = - BaseMavenRepository.ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext) + BaseMavenRepository.ivyLikePath( + dependency.module.organization, + dependency.module.name, + project.version, + subDir, + baseSuffix, + ext + ) val path = if (ivyLike) @@ -176,7 +193,7 @@ object BaseMavenRepository { Map.empty, dependency.attributes ) - .withDefaultChecksums + .withDefaultChecksums if (dependency.attributes.`type` == "jar") { artifact = artifact.withDefaultSignature @@ -188,20 +205,22 @@ object BaseMavenRepository { artifact .copy(extra = artifact.extra ++ Map( - Artifact.sourcesMd5 -> (srcPath + ".md5"), - Artifact.sourcesSha1 -> (srcPath + ".sha1"), - Artifact.sources -> srcPath, - Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), - Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), - Artifact.sourcesSig -> (srcPath + ".asc"), - Artifact.javadocMd5 -> (javadocPath + ".md5"), - Artifact.javadocSha1 -> (javadocPath + ".sha1"), - Artifact.javadoc -> javadocPath, - Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), - Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), - Artifact.javadocSig -> (javadocPath + ".asc") + Artifact.sourcesMd5 -> (srcPath + ".md5"), + Artifact.sourcesSha1 -> (srcPath + ".sha1"), + Artifact.sources -> srcPath, + Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), + Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), + Artifact.sourcesSig -> (srcPath + ".asc"), + Artifact.javadocMd5 -> (javadocPath + ".md5"), + Artifact.javadocSha1 -> (javadocPath + ".sha1"), + Artifact.javadoc -> javadocPath, + Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), + Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), + Artifact.javadocSig -> (javadocPath + ".asc") )) - } else artifact.withJavadocSources + } else + artifact + .withJavadocSources } Seq(artifact) @@ -226,6 +245,7 @@ abstract class BaseMavenRepository( val source = BaseMavenRepository.Source(root, ivyLike) def projectArtifact(module: Module, version: String): Artifact = { + val path = ( if (ivyLike) ivyLikePath(module.organization, module.name, version, "poms", "", "pom") @@ -269,8 +289,10 @@ abstract class BaseMavenRepository( Some(artifact) } - def versions(module: Module, - cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Versions] = { + def versions( + module: Module, + cachePolicy: CachePolicy = CachePolicy.Default + ): EitherT[Task, String, Versions] = { EitherT( versionsArtifact(module) match { @@ -290,9 +312,11 @@ abstract class BaseMavenRepository( ) } - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { + def findNoInterval( + module: Module, + version: String, + cachePolicy: CachePolicy + ): EitherT[Task, String, Project] = { EitherT { fetch(projectArtifact(module, version), cachePolicy) @@ -308,37 +332,41 @@ abstract class BaseMavenRepository( } } - def find(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, (Artifact.Source, Project)] = { + def find( + module: Module, + version: String, + cachePolicy: CachePolicy + ): EitherT[Task, String, (Artifact.Source, Project)] = { - Parse.versionInterval(version).filter(_.isValid) match { - case None => findNoInterval(module, version, cachePolicy).map((source, _)) - case Some(itv) => - versions(module, cachePolicy) - .flatMap { versions0 => - val eitherVersion = { - val release = Version(versions0.release) + Parse.versionInterval(version) + .filter(_.isValid) match { + case None => + findNoInterval(module, version, cachePolicy).map((source, _)) + case Some(itv) => + versions(module, cachePolicy) + .flatMap { versions0 => + val eitherVersion = { + val release = Version(versions0.release) - if (itv.contains(release)) \/-(versions0.release) - else { - val inInterval = versions0.available - .map(Version(_)) - .filter(itv.contains) + if (itv.contains(release)) \/-(versions0.release) + else { + val inInterval = versions0.available + .map(Version(_)) + .filter(itv.contains) - if (inInterval.isEmpty) -\/(s"No version found for $version") - else \/-(inInterval.max.repr) + if (inInterval.isEmpty) -\/(s"No version found for $version") + else \/-(inInterval.max.repr) + } + } + + eitherVersion match { + case -\/(reason) => EitherT[Task, String, (Artifact.Source, Project)](Task.now(-\/(reason))) + case \/-(version0) => + findNoInterval(module, version0, cachePolicy) + .map(_.copy(versions = Some(versions0))) + .map((source, _)) } } - - eitherVersion match { - case -\/(reason) => EitherT[Task, String, (Artifact.Source, Project)](Task.now(-\/(reason))) - case \/-(version0) => - findNoInterval(module, version0, cachePolicy) - .map(_.copy(versions = Some(versions0))) - .map((source, _)) - } - } } } From 6f68abc4d6bf586acbc2cc325f3da147c788ec35 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:36 +0100 Subject: [PATCH 03/13] Formatting / styling, minor refactoring --- .../main/scala/coursier/cli/Coursier.scala | 4 +-- files/src/main/scala/coursier/Files.scala | 32 +++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 0162ebd44..dffa563b6 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -29,8 +29,8 @@ case class Coursier( def fileRepr(f: File) = f.toString - val logger: MavenRepository.Logger with FilesLogger = - new MavenRepository.Logger with FilesLogger { + val logger: MavenRepository.Logger with Files.Logger = + new MavenRepository.Logger with Files.Logger { def println(s: String) = Console.err.println(s) def downloading(url: String) = diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index ad707f46b..f19344080 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -1,28 +1,25 @@ package coursier -import java.net.{URI, URL} +import java.net.{ URI, URL } import coursier.core.Repository.CachePolicy import scala.annotation.tailrec -import scalaz.{-\/, \/-, \/, EitherT} +import scalaz.{ -\/, \/-, \/, EitherT } import scalaz.concurrent.Task import java.io._ -// FIXME This kind of side-effecting API is lame, we should aim at a more functional one. -trait FilesLogger { - def foundLocally(f: File): Unit - def downloadingArtifact(url: String): Unit - def downloadedArtifact(url: String, success: Boolean): Unit -} +case class Files( + cache: Seq[(String, File)], + tmp: () => File, + logger: Option[Files.Logger] = None +) { -case class Files(cache: Seq[(String, File)], - tmp: () => File, - logger: Option[FilesLogger] = None) { - - def file(artifact: Artifact, - cachePolicy: CachePolicy): EitherT[Task, String, File] = { + def file( + artifact: Artifact, + cachePolicy: CachePolicy + ): EitherT[Task, String, File] = { if (artifact.url.startsWith("file:///")) { val f = new File(new URI(artifact.url) .getPath) @@ -100,6 +97,13 @@ case class Files(cache: Seq[(String, File)], object Files { + // FIXME This kind of side-effecting API is lame, we should aim at a more functional one. + trait Logger { + def foundLocally(f: File): Unit + def downloadingArtifact(url: String): Unit + def downloadedArtifact(url: String, success: Boolean): Unit + } + var bufferSize = 1024*1024 def readFullySync(is: InputStream) = { From e2c654eb888cc603da649ad5d0cd748b10423325 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:37 +0100 Subject: [PATCH 04/13] Move back CachePolicy to namespace coursier.core --- .../main/scala/coursier/cli/Coursier.scala | 2 +- .../scala/coursier/core/MavenRepository.scala | 2 +- .../scala/coursier/core/MavenRepository.scala | 2 +- .../main/scala/coursier/core/Repository.scala | 80 +++++++++---------- core/src/main/scala/coursier/package.scala | 3 + .../scala/coursier/test/TestRepository.scala | 2 +- files/src/main/scala/coursier/Files.scala | 2 - 7 files changed, 46 insertions(+), 47 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index dffa563b6..d4d00a449 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -4,7 +4,7 @@ package cli import java.io.File import caseapp._ -import coursier.core.{MavenRepository, Parse, Repository}, Repository.CachePolicy +import coursier.core.{ MavenRepository, Parse, Repository, CachePolicy } import scalaz.concurrent.Task diff --git a/core-js/src/main/scala/coursier/core/MavenRepository.scala b/core-js/src/main/scala/coursier/core/MavenRepository.scala index 7215df392..67cf122a7 100644 --- a/core-js/src/main/scala/coursier/core/MavenRepository.scala +++ b/core-js/src/main/scala/coursier/core/MavenRepository.scala @@ -93,7 +93,7 @@ case class MavenRepository( def fetch( artifact: Artifact, - cachePolicy: Repository.CachePolicy + cachePolicy: CachePolicy ): EitherT[Task, String, String] = { val url0 = root + artifact.url diff --git a/core-jvm/src/main/scala/coursier/core/MavenRepository.scala b/core-jvm/src/main/scala/coursier/core/MavenRepository.scala index c9cc2d97c..5b39acf82 100644 --- a/core-jvm/src/main/scala/coursier/core/MavenRepository.scala +++ b/core-jvm/src/main/scala/coursier/core/MavenRepository.scala @@ -19,7 +19,7 @@ case class MavenRepository( def fetch( artifact: Artifact, - cachePolicy: Repository.CachePolicy + cachePolicy: CachePolicy ): EitherT[Task, String, String] = { def locally(eitherFile: String \/ File) = { diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index ef35e01db..deb8eed92 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -1,7 +1,5 @@ package coursier.core -import coursier.core.Repository.CachePolicy - import scalaz.{ -\/, \/-, \/, EitherT } import scalaz.concurrent.Task @@ -11,7 +9,7 @@ trait Repository { def find( module: Module, version: String, - cachePolicy: Repository.CachePolicy = Repository.CachePolicy.Default + cachePolicy: CachePolicy = CachePolicy.Default ): EitherT[Task, String, (Artifact.Source, Project)] } @@ -72,43 +70,6 @@ object Repository { } } - sealed trait CachePolicy { - def apply[E,T](local: => Task[E \/ T]) - (remote: => Task[E \/ T]): Task[E \/ T] - - def saving[E,T](local: => Task[E \/ T]) - (remote: => Task[E \/ T]) - (save: => T => Task[Unit]): Task[E \/ T] = - apply(local)(CachePolicy.saving(remote)(save)) - } - - object CachePolicy { - def saving[E,T](remote: => Task[E \/ T]) - (save: T => Task[Unit]): Task[E \/ T] = { - for { - res <- remote - _ <- res.fold(_ => Task.now(()), t => save(t)) - } yield res - } - - case object Default extends CachePolicy { - def apply[E,T](local: => Task[E \/ T]) - (remote: => Task[E \/ T]): Task[E \/ T] = - local - .flatMap(res => if (res.isLeft) remote else Task.now(res)) - } - case object LocalOnly extends CachePolicy { - def apply[E,T](local: => Task[E \/ T]) - (remote: => Task[E \/ T]): Task[E \/ T] = - local - } - case object ForceDownload extends CachePolicy { - def apply[E,T](local: => Task[E \/ T]) - (remote: => Task[E \/ T]): Task[E \/ T] = - remote - } - } - implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { def withDefaultChecksums: Artifact = underlying.copy(extra = underlying.extra ++ Seq( @@ -370,4 +331,41 @@ abstract class BaseMavenRepository( } } -} \ No newline at end of file +} + +sealed trait CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] + + def saving[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]) + (save: => T => Task[Unit]): Task[E \/ T] = + apply(local)(CachePolicy.saving(remote)(save)) +} + +object CachePolicy { + def saving[E,T](remote: => Task[E \/ T]) + (save: T => Task[Unit]): Task[E \/ T] = { + for { + res <- remote + _ <- res.fold(_ => Task.now(()), t => save(t)) + } yield res + } + + case object Default extends CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] = + local + .flatMap(res => if (res.isLeft) remote else Task.now(res)) + } + case object LocalOnly extends CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] = + local + } + case object ForceDownload extends CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] = + remote + } +} diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index dcbeeec53..7ba3175bb 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -52,6 +52,9 @@ package object coursier { type Scope = core.Scope val Scope: core.Scope.type = core.Scope + type CachePolicy = core.CachePolicy + val CachePolicy: core.CachePolicy.type = core.CachePolicy + type Repository = core.Repository val Repository: core.Repository.type = core.Repository diff --git a/core/src/test/scala/coursier/test/TestRepository.scala b/core/src/test/scala/coursier/test/TestRepository.scala index 80afb0bb6..8a91d6feb 100644 --- a/core/src/test/scala/coursier/test/TestRepository.scala +++ b/core/src/test/scala/coursier/test/TestRepository.scala @@ -11,7 +11,7 @@ class TestRepository(projects: Map[(Module, String), Project]) extends Repositor val source = new core.Artifact.Source { def artifacts(dependency: Dependency, project: Project) = ??? } - def find(module: Module, version: String, cachePolicy: Repository.CachePolicy) = + def find(module: Module, version: String, cachePolicy: CachePolicy) = EitherT(Task.now( projects.get((module, version)).map((source, _)).toRightDisjunction("Not found") )) diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index f19344080..46b337a9b 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -2,8 +2,6 @@ package coursier import java.net.{ URI, URL } -import coursier.core.Repository.CachePolicy - import scala.annotation.tailrec import scalaz.{ -\/, \/-, \/, EitherT } import scalaz.concurrent.Task From 6eeb792829888fee6fbafd89f6e9f35db0c511c2 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:39 +0100 Subject: [PATCH 05/13] Made implicit the cache policy arguments --- .../main/scala/coursier/cli/Coursier.scala | 46 +++++++++++++------ .../test/scala/coursier/test/JsTests.scala | 2 + .../main/scala/coursier/core/Repository.scala | 10 ++-- core/src/main/scala/coursier/package.scala | 2 + .../scala/coursier/test/CentralTests.scala | 2 + .../scala/coursier/test/ResolutionTests.scala | 2 + .../scala/coursier/test/TestRepository.scala | 2 +- web/src/main/scala/coursier/web/Backend.scala | 4 +- 8 files changed, 51 insertions(+), 19 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index d4d00a449..e0d391a20 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -6,12 +6,14 @@ import java.io.File import caseapp._ import coursier.core.{ MavenRepository, Parse, Repository, CachePolicy } +import scalaz.{\/-, -\/} import scalaz.concurrent.Task case class Coursier( scope: List[String], keepOptional: Boolean, fetch: Boolean, + @ExtraName("P") @ExtraName("cp") classpath: Boolean, @ExtraName("c") offline: Boolean, @ExtraName("v") verbose: List[Unit], @ExtraName("N") maxIterations: Int = 100 @@ -29,10 +31,11 @@ case class Coursier( def fileRepr(f: File) = f.toString + def println(s: String) = Console.err.println(s) + + val logger: MavenRepository.Logger with Files.Logger = new MavenRepository.Logger with Files.Logger { - def println(s: String) = Console.err.println(s) - def downloading(url: String) = println(s"Downloading $url") def downloaded(url: String, success: Boolean) = @@ -57,6 +60,12 @@ case class Coursier( ) } + implicit val cachePolicy = + if (offline) + CachePolicy.LocalOnly + else + CachePolicy.Default + val cachedMavenCentral = Repository.mavenCentral.copy( cache = Some(centralCacheDir), logger = if (verbose0 <= 1) None else Some(logger) @@ -78,7 +87,7 @@ case class Coursier( } if (malformed.nonEmpty) { - Console.err.println(s"Malformed dependencies:\n${malformed.map(_.mkString(":")).mkString("\n")}") + println(s"Malformed dependencies:\n${malformed.map(_.mkString(":")).mkString("\n")}") sys exit 1 } @@ -114,7 +123,7 @@ case class Coursier( .run if (!res.isDone) { - Console.err.println(s"Maximum number of iteration reached!") + println(s"Maximum number of iteration reached!") sys exit 1 } @@ -145,14 +154,8 @@ case class Coursier( } } - if (fetch) { - println() - - val cachePolicy: CachePolicy = - if (offline) - CachePolicy.LocalOnly - else - CachePolicy.Default + if (fetch || classpath) { + println("") val artifacts = res.artifacts @@ -164,10 +167,25 @@ case class Coursier( if (verbose0 <= 0) None else Some(logger) ) - val tasks = artifacts.map(files.file(_, cachePolicy).run) + val tasks = artifacts.map(artifact => files.file(artifact, cachePolicy).run.map(artifact.->)) val task = Task.gatherUnordered(tasks) - task.run + val results = task.run + val errors = results.collect{case (artifact, -\/(err)) => artifact -> err } + val files0 = results.collect{case (artifact, \/-(f)) => f } + + if (errors.nonEmpty) { + println(s"${errors.size} error(s):") + for ((artifact, error) <- errors) { + println(s" ${artifact.url}: $error") + } + } + + Console.out.println( + files0 + .map(_.toString) + .mkString(File.pathSeparator) + ) } } diff --git a/core-js/src/test/scala/coursier/test/JsTests.scala b/core-js/src/test/scala/coursier/test/JsTests.scala index d4747b62c..c59f1dc80 100644 --- a/core-js/src/test/scala/coursier/test/JsTests.scala +++ b/core-js/src/test/scala/coursier/test/JsTests.scala @@ -26,6 +26,8 @@ object JsTests extends TestSuite { } 'getProj{ + implicit val cachePolicy = CachePolicy.Default + Repository.mavenCentral .find(Module("ch.qos.logback", "logback-classic"), "1.1.3") .map{case (_, proj) => diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index deb8eed92..2a99a088f 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -8,8 +8,9 @@ import coursier.core.compatibility.encodeURIComponent trait Repository { def find( module: Module, - version: String, - cachePolicy: CachePolicy = CachePolicy.Default + version: String + )(implicit + cachePolicy: CachePolicy ): EitherT[Task, String, (Artifact.Source, Project)] } @@ -38,6 +39,8 @@ object Repository { repositories: Seq[Repository], module: Module, version: String + )(implicit + cachePolicy: CachePolicy ): EitherT[Task, Seq[String], (Artifact.Source, Project)] = { val lookups = repositories @@ -295,7 +298,8 @@ abstract class BaseMavenRepository( def find( module: Module, - version: String, + version: String + )(implicit cachePolicy: CachePolicy ): EitherT[Task, String, (Artifact.Source, Project)] = { diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 7ba3175bb..f3c3a58f6 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -94,6 +94,8 @@ package object coursier { implicit def fetch( repositories: Seq[core.Repository] + )(implicit + cachePolicy: CachePolicy ): ResolutionProcess.Fetch[Task] = { modVers => diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 10e7cb5b4..82ee9cd62 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -13,6 +13,8 @@ object CentralTests extends TestSuite { Repository.mavenCentral ) + implicit val cachePolicy = CachePolicy.Default + def resolve(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None, extraRepo: Option[Repository] = None) = { val repositories0 = extraRepo.toSeq ++ repositories diff --git a/core/src/test/scala/coursier/test/ResolutionTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala index c88266e18..6313d3fdd 100644 --- a/core/src/test/scala/coursier/test/ResolutionTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -9,6 +9,8 @@ import coursier.test.compatibility._ object ResolutionTests extends TestSuite { + implicit val cachePolicy = CachePolicy.Default + def resolve0(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { Resolution(deps, filter = filter) .process diff --git a/core/src/test/scala/coursier/test/TestRepository.scala b/core/src/test/scala/coursier/test/TestRepository.scala index 8a91d6feb..18b202e6f 100644 --- a/core/src/test/scala/coursier/test/TestRepository.scala +++ b/core/src/test/scala/coursier/test/TestRepository.scala @@ -11,7 +11,7 @@ class TestRepository(projects: Map[(Module, String), Project]) extends Repositor val source = new core.Artifact.Source { def artifacts(dependency: Dependency, project: Project) = ??? } - def find(module: Module, version: String, cachePolicy: CachePolicy) = + def find(module: Module, version: String)(implicit cachePolicy: CachePolicy) = EitherT(Task.now( projects.get((module, version)).map((source, _)).toRightDisjunction("Not found") )) diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 708ba73c7..61270b88b 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -133,9 +133,11 @@ class Backend($: BackendScope[Unit, State]) { filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) ) + implicit val cachePolicy = CachePolicy.Default + res .process - .run(fetch(s.repositories.map(r => r.copy(logger = Some(logger)))), 100) + .run(s.repositories.map(r => r.copy(logger = Some(logger))), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context From be3fa1b94f39ef62a1777c94319428a1a874021d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:40 +0100 Subject: [PATCH 06/13] Better repository management --- .../main/scala/coursier/cli/Coursier.scala | 107 ++++++++++---- .../scala/coursier/cli/Repositories.scala | 87 +++++++++++ files/src/main/scala/coursier/Cache.scala | 137 ++++++++++++++++++ 3 files changed, 303 insertions(+), 28 deletions(-) create mode 100644 cli/src/main/scala/coursier/cli/Repositories.scala create mode 100644 files/src/main/scala/coursier/Cache.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index e0d391a20..1fc1357cc 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -4,9 +4,9 @@ package cli import java.io.File import caseapp._ -import coursier.core.{ MavenRepository, Parse, Repository, CachePolicy } +import coursier.core.{ MavenRepository, Parse, CachePolicy } -import scalaz.{\/-, -\/} +import scalaz.{ \/-, -\/ } import scalaz.concurrent.Task case class Coursier( @@ -16,7 +16,8 @@ case class Coursier( @ExtraName("P") @ExtraName("cp") classpath: Boolean, @ExtraName("c") offline: Boolean, @ExtraName("v") verbose: List[Unit], - @ExtraName("N") maxIterations: Int = 100 + @ExtraName("N") maxIterations: Int = 100, + @ExtraName("r") repository: List[String] ) extends App { val verbose0 = verbose.length @@ -26,15 +27,30 @@ case class Coursier( else scope.map(Parse.scope) val scopes = scopes0.toSet - val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/metadata/central") - val centralFilesCacheDir = new File(sys.props("user.home") + "/.coursier/cache/files/central") - def fileRepr(f: File) = f.toString def println(s: String) = Console.err.println(s) - val logger: MavenRepository.Logger with Files.Logger = + def defaultLogger: MavenRepository.Logger with Files.Logger = + new MavenRepository.Logger with Files.Logger { + def downloading(url: String) = + println(s"Downloading $url") + def downloaded(url: String, success: Boolean) = + if (!success) + println(s"Failed to download $url") + def readingFromCache(f: File) = {} + def puttingInCache(f: File) = {} + + def foundLocally(f: File) = {} + def downloadingArtifact(url: String) = + println(s"Downloading $url") + def downloadedArtifact(url: String, success: Boolean) = + if (!success) + println(s"Failed to download $url") + } + + def verboseLogger: MavenRepository.Logger with Files.Logger = new MavenRepository.Logger with Files.Logger { def downloading(url: String) = println(s"Downloading $url") @@ -60,22 +76,54 @@ case class Coursier( ) } + val logger = if (verbose0 <= 0) defaultLogger else verboseLogger + implicit val cachePolicy = if (offline) CachePolicy.LocalOnly else CachePolicy.Default - val cachedMavenCentral = Repository.mavenCentral.copy( - cache = Some(centralCacheDir), - logger = if (verbose0 <= 1) None else Some(logger) - ) - val repositories = Seq[Repository]( - cachedMavenCentral, - Repository.ivy2Local.copy( - logger = if (verbose0 <= 1) None else Some(logger) + val cache = Cache.default + + val repositoryIds = { + val repository0 = repository + .flatMap(_.split(',')) + .map(_.trim) + .filter(_.nonEmpty) + + if (repository0.isEmpty) + cache.default() + else + repository0 + } + + val existingRepo = cache + .list() + .map(_._1) + .toSet + if (repositoryIds.exists(!existingRepo(_))) { + val notFound = repositoryIds + .filter(!existingRepo(_)) + + Console.err.println( + (if (notFound.lengthCompare(1) == 1) "Repository" else "Repositories") + + " not found: " + + notFound.mkString(", ") ) - ) + + sys.exit(1) + } + + + val (repositories0, fileCaches) = cache + .list() + .map{case (_, repo, cacheEntry) => (repo, cacheEntry)} + .unzip + + val repositories = repositories0 + .map(_.copy(logger = Some(logger))) + val (splitDependencies, malformed) = remainingArgs.toList .map(_.split(":", 3).toSeq) @@ -159,13 +207,9 @@ case class Coursier( val artifacts = res.artifacts - val files = new Files( - Seq( - cachedMavenCentral.root -> centralFilesCacheDir - ), - () => ???, - if (verbose0 <= 0) None else Some(logger) - ) + val files = cache + .files() + .copy(logger = Some(logger)) val tasks = artifacts.map(artifact => files.file(artifact, cachePolicy).run.map(artifact.->)) val task = Task.gatherUnordered(tasks) @@ -181,11 +225,18 @@ case class Coursier( } } - Console.out.println( - files0 - .map(_.toString) - .mkString(File.pathSeparator) - ) + if (classpath) + Console.out.println( + files0 + .map(_.toString) + .mkString(File.pathSeparator) + ) + else + println( + files0 + .map(_.toString) + .mkString("\n") + ) } } diff --git a/cli/src/main/scala/coursier/cli/Repositories.scala b/cli/src/main/scala/coursier/cli/Repositories.scala new file mode 100644 index 000000000..37903016f --- /dev/null +++ b/cli/src/main/scala/coursier/cli/Repositories.scala @@ -0,0 +1,87 @@ +package coursier.cli + +import coursier.Cache +import caseapp._ + +// TODO: allow removing a repository (with confirmations, etc.) +case class Repositories( + @ValueDescription("id:baseUrl") @ExtraName("a") add: List[String], + @ExtraName("L") list: Boolean, + @ExtraName("l") defaultList: Boolean, + ivyLike: Boolean +) extends App { + + if (add.exists(!_.contains(":"))) { + CaseApp.printUsage[Repositories](err = true) + sys.exit(255) + } + + val add0 = add + .map{ s => + val Seq(id, baseUrl) = s.split(":", 2).toSeq + id -> baseUrl + } + + if ( + add0.exists(_._1.contains("/")) || + add0.exists(_._1.startsWith(".")) || + add0.exists(_._1.isEmpty) + ) { + CaseApp.printUsage[Repositories](err = true) + sys.exit(255) + } + + + val cache = Cache.default + + if (cache.cache.exists() && !cache.cache.isDirectory) { + Console.err.println(s"Error: ${cache.cache} not a directory") + sys.exit(1) + } + + if (!cache.cache.exists()) + cache.init() + + val current = cache.list().map(_._1).toSet + + val alreadyAdded = add0 + .map(_._1) + .filter(current) + + if (alreadyAdded.nonEmpty) { + Console.err.println(s"Error: already added: ${alreadyAdded.mkString(", ")}") + sys.exit(1) + } + + for ((id, baseUrl0) <- add0) { + val baseUrl = + if (baseUrl0.endsWith("/")) + baseUrl0 + else + baseUrl0 + "/" + + cache.add(id, baseUrl, ivyLike = ivyLike) + } + + if (defaultList) { + val map = cache.repositoryMap() + + for (id <- cache.default(withNotFound = true)) + map.get(id) match { + case Some(repo) => + println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) + case None => + println(s"$id (not found)") + } + } + + if (list) + for ((id, repo, _) <- cache.list().sortBy(_._1)) { + println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) + } + +} + +object Repositories extends AppOf[Repositories] { + val parser = default +} diff --git a/files/src/main/scala/coursier/Cache.scala b/files/src/main/scala/coursier/Cache.scala new file mode 100644 index 000000000..ebd80eff9 --- /dev/null +++ b/files/src/main/scala/coursier/Cache.scala @@ -0,0 +1,137 @@ +package coursier + +import java.io.{PrintWriter, File} + +import coursier.core.MavenRepository + +import scala.io.Source + +object Cache { + + def mavenRepository(lines: Seq[String]): Option[MavenRepository] = { + def isMaven = + lines + .find(_.startsWith("maven:")) + .map(_.stripPrefix("maven:").trim) + .contains("true") + + def ivyLike = + lines + .find(_.startsWith("ivy-like:")) + .map(_.stripPrefix("ivy-like:").trim) + .contains("true") + + def base = + lines + .find(_.startsWith("base:")) + .map(_.stripPrefix("base:").trim) + .filter(_.nonEmpty) + + if (isMaven) + base.map(MavenRepository(_, ivyLike = ivyLike)) + else + None + } + + lazy val default = Cache(new File(sys.props("user.home") + "/.coursier/cache")) + +} + +case class Cache(cache: File) { + + import Cache._ + + lazy val repoDir = new File(cache, "repositories") + lazy val metadataBase = new File(cache, "metadata") + lazy val fileBase = new File(cache, "files") + + lazy val defaultFile = new File(repoDir, ".default") + + def add(id: String, base: String, ivyLike: Boolean): Unit = { + repoDir.mkdirs() + val f = new File(repoDir, id) + val w = new PrintWriter(f) + try w.println((Seq("maven: true", s"base: $base") ++ (if (ivyLike) Seq("ivy-like: true") else Nil)).mkString("\n")) + finally w.close() + } + + def addCentral(): Unit = + add("central", "https://repo1.maven.org/maven2/", ivyLike = false) + + def addIvy2Local(): Unit = + add("ivy2local", "file://" + sys.props("user.home") + "/.ivy2/local/", ivyLike = true) + + def init(ifEmpty: Boolean = true): Unit = + if (!ifEmpty || !cache.exists()) { + repoDir.mkdirs() + metadataBase.mkdirs() + fileBase.mkdirs() + addCentral() + addIvy2Local() + setDefault("ivy2local", "central") + } + + def setDefault(ids: String*): Unit = { + defaultFile.getParentFile.mkdirs() + val w = new PrintWriter(defaultFile) + try w.println(ids.mkString("\n")) + finally w.close() + } + + def list(): Seq[(String, MavenRepository, (String, File))] = + Option(repoDir.listFiles()) + .map(_.toSeq) + .getOrElse(Nil) + .filter(f => f.isFile && !f.getName.startsWith(".")) + .flatMap { f => + val name = f.getName + val lines = Source.fromFile(f).getLines().toList + mavenRepository(lines) + .map(repo => + (name, repo.copy(cache = Some(new File(metadataBase, name))), (repo.root, new File(fileBase, name))) + ) + } + + def map(): Map[String, (MavenRepository, (String, File))] = + list() + .map{case (id, repo, fileCache) => id -> (repo, fileCache) } + .toMap + + + def repositories(): Seq[MavenRepository] = + list().map(_._2) + + def repositoryMap(): Map[String, MavenRepository] = + list() + .map{case (id, repo, _) => id -> repo} + .toMap + + def fileCaches(): Seq[(String, File)] = + list().map(_._3) + + def default(withNotFound: Boolean = false): Seq[String] = + if (defaultFile.exists()) { + val default0 = + Source.fromFile(defaultFile) + .getLines() + .map(_.trim) + .filter(_.nonEmpty) + .toList + + val found = list() + .map(_._1) + .toSet + + default0 + .filter(found) + } else + Nil + + def files(): Files = { + val map0 = map() + val default0 = default() + + new Files(default0.map(map0(_)._2), () => ???) + } + +} From 6ed8f6e2d137299175cb22b0b21650f7441d63dc Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:41 +0100 Subject: [PATCH 07/13] Minor refactoring --- .../main/scala/coursier/core/Repository.scala | 138 +++++++++--------- web/src/main/scala/coursier/web/Backend.scala | 10 +- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index 2a99a088f..68a98e6ea 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -105,6 +105,74 @@ object Repository { } } +case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { + import Repository._ + + def artifacts( + dependency: Dependency, + project: Project + ): Seq[Artifact] = { + + def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = + BaseMavenRepository.ivyLikePath( + dependency.module.organization, + dependency.module.name, + project.version, + subDir, + baseSuffix, + ext + ) + + val path = + if (ivyLike) + ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) + else + dependency.module.organization.split('.').toSeq ++ Seq( + dependency.module.name, + project.version, + s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + ) + + var artifact = + Artifact( + root + path.mkString("/"), + Map.empty, + dependency.attributes + ) + .withDefaultChecksums + + if (dependency.attributes.`type` == "jar") { + artifact = artifact.withDefaultSignature + + artifact = + if (ivyLike) { + val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") + val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") + + artifact + .copy(extra = artifact.extra ++ Map( + Artifact.sourcesMd5 -> (srcPath + ".md5"), + Artifact.sourcesSha1 -> (srcPath + ".sha1"), + Artifact.sources -> srcPath, + Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), + Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), + Artifact.sourcesSig -> (srcPath + ".asc"), + Artifact.javadocMd5 -> (javadocPath + ".md5"), + Artifact.javadocSha1 -> (javadocPath + ".sha1"), + Artifact.javadoc -> javadocPath, + Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), + Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), + Artifact.javadocSig -> (javadocPath + ".asc") + )) + } else + artifact + .withJavadocSources + } + + Seq(artifact) + } +} + object BaseMavenRepository { def ivyLikePath( @@ -123,74 +191,6 @@ object BaseMavenRepository { s"$name$baseSuffix.$ext" ) - case class Source(root: String, ivyLike: Boolean) extends Artifact.Source { - import Repository._ - - def artifacts( - dependency: Dependency, - project: Project - ): Seq[Artifact] = { - - def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = - BaseMavenRepository.ivyLikePath( - dependency.module.organization, - dependency.module.name, - project.version, - subDir, - baseSuffix, - ext - ) - - val path = - if (ivyLike) - ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) - else - dependency.module.organization.split('.').toSeq ++ Seq( - dependency.module.name, - project.version, - s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" - ) - - var artifact = - Artifact( - root + path.mkString("/"), - Map.empty, - dependency.attributes - ) - .withDefaultChecksums - - if (dependency.attributes.`type` == "jar") { - artifact = artifact.withDefaultSignature - - artifact = - if (ivyLike) { - val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") - val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") - - artifact - .copy(extra = artifact.extra ++ Map( - Artifact.sourcesMd5 -> (srcPath + ".md5"), - Artifact.sourcesSha1 -> (srcPath + ".sha1"), - Artifact.sources -> srcPath, - Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), - Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), - Artifact.sourcesSig -> (srcPath + ".asc"), - Artifact.javadocMd5 -> (javadocPath + ".md5"), - Artifact.javadocSha1 -> (javadocPath + ".sha1"), - Artifact.javadoc -> javadocPath, - Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), - Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), - Artifact.javadocSig -> (javadocPath + ".asc") - )) - } else - artifact - .withJavadocSources - } - - Seq(artifact) - } - } - } abstract class BaseMavenRepository( @@ -206,7 +206,7 @@ abstract class BaseMavenRepository( import Repository._ import BaseMavenRepository._ - val source = BaseMavenRepository.Source(root, ivyLike) + val source = MavenSource(root, ivyLike) def projectArtifact(module: Module, version: String): Artifact = { diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 61270b88b..b0b4dc027 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,10 +1,10 @@ package coursier package web -import coursier.core.{Repository, BaseMavenRepository, MavenRepository} -import japgolly.scalajs.react.vdom.{TagMod, Attr} +import coursier.core.{ Repository, MavenRepository, MavenSource } +import japgolly.scalajs.react.vdom.{ TagMod, Attr } import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml -import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope} +import japgolly.scalajs.react.{ ReactEventI, ReactComponentB, BackendScope } import japgolly.scalajs.react.vdom.prefix_<^._ import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue import org.scalajs.jquery.jQuery @@ -12,7 +12,7 @@ import org.scalajs.jquery.jQuery import scala.concurrent.Future import scala.scalajs.js -import js.Dynamic.{global => g} +import js.Dynamic.{ global => g } case class ResolutionOptions(followOptional: Boolean = false, keepTest: Boolean = false) @@ -248,7 +248,7 @@ object App { )), <.td(Seq[Seq[TagMod]]( res.projectCache.get(dep.moduleVersion) match { - case Some((source: BaseMavenRepository.Source, proj)) if !source.ivyLike => + case Some((source: MavenSource, proj)) if !source.ivyLike => // FIXME Maven specific, generalize with source.artifacts val version0 = finalVersionOpt getOrElse dep.version val relPath = From 8e8bbc6c9fb35e603cd1e243c48f4b5010e6aeed Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:42 +0100 Subject: [PATCH 08/13] Changes in artifact definition --- .../scala/coursier/core/Definitions.scala | 21 +------ .../main/scala/coursier/core/Repository.scala | 59 ++++++++----------- 2 files changed, 26 insertions(+), 54 deletions(-) diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index 76a27ae66..504bf09af 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -101,29 +101,12 @@ object Versions { case class Artifact( url: String, - extra: Map[String, String], + checksumUrls: Map[String, String], + extra: Map[String, Artifact], attributes: Attributes ) object Artifact { - val md5 = "md5" - val sha1 = "sha1" - val sig = "pgp" - val sigMd5 = "md5-pgp" - val sigSha1 = "sha1-pgp" - val sources = "src" - val sourcesMd5 = "md5-src" - val sourcesSha1 = "sha1-src" - val sourcesSig = "src-pgp" - val sourcesSigMd5 = "md5-src-pgp" - val sourcesSigSha1 = "sha1-src-pgp" - val javadoc = "javadoc" - val javadocMd5 = "md5-javadoc" - val javadocSha1 = "sha1-javadoc" - val javadocSig = "javadoc-pgp" - val javadocSigMd5 = "md5-javadoc-pgp" - val javadocSigSha1 = "sha1-javadoc-pgp" - trait Source { def artifacts(dependency: Dependency, project: Project): Seq[Artifact] } diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index 68a98e6ea..a1263f75c 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -75,31 +75,25 @@ object Repository { implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { def withDefaultChecksums: Artifact = - underlying.copy(extra = underlying.extra ++ Seq( - Artifact.md5 -> (underlying.url + ".md5"), - Artifact.sha1 -> (underlying.url + ".sha1") + underlying.copy(checksumUrls = underlying.checksumUrls ++ Seq( + "md5" -> (underlying.url + ".md5"), + "sha1" -> (underlying.url + ".sha1") )) def withDefaultSignature: Artifact = underlying.copy(extra = underlying.extra ++ Seq( - Artifact.sigMd5 -> (underlying.url + ".asc.md5"), - Artifact.sigSha1 -> (underlying.url + ".asc.sha1"), - Artifact.sig -> (underlying.url + ".asc") + "sig" -> + Artifact(underlying.url + ".asc", Map.empty, Map.empty, Attributes("asc", "")) + .withDefaultChecksums )) def withJavadocSources: Artifact = { val base = underlying.url.stripSuffix(".jar") underlying.copy(extra = underlying.extra ++ Seq( - Artifact.sourcesMd5 -> (base + "-sources.jar.md5"), - Artifact.sourcesSha1 -> (base + "-sources.jar.sha1"), - Artifact.sources -> (base + "-sources.jar"), - Artifact.sourcesSigMd5 -> (base + "-sources.jar.asc.md5"), - Artifact.sourcesSigSha1 -> (base + "-sources.jar.asc.sha1"), - Artifact.sourcesSig -> (base + "-sources.jar.asc"), - Artifact.javadocMd5 -> (base + "-javadoc.jar.md5"), - Artifact.javadocSha1 -> (base + "-javadoc.jar.sha1"), - Artifact.javadoc -> (base + "-javadoc.jar"), - Artifact.javadocSigMd5 -> (base + "-javadoc.jar.asc.md5"), - Artifact.javadocSigSha1 -> (base + "-javadoc.jar.asc.sha1"), - Artifact.javadocSig -> (base + "-javadoc.jar.asc") + "sources" -> Artifact(base + "-sources.jar", Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? + .withDefaultChecksums + .withDefaultSignature, + "javadoc" -> Artifact(base + "-javadoc.jar", Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above + .withDefaultChecksums + .withDefaultSignature )) } } @@ -137,6 +131,7 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { Artifact( root + path.mkString("/"), Map.empty, + Map.empty, dependency.attributes ) .withDefaultChecksums @@ -150,19 +145,14 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") artifact - .copy(extra = artifact.extra ++ Map( - Artifact.sourcesMd5 -> (srcPath + ".md5"), - Artifact.sourcesSha1 -> (srcPath + ".sha1"), - Artifact.sources -> srcPath, - Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), - Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), - Artifact.sourcesSig -> (srcPath + ".asc"), - Artifact.javadocMd5 -> (javadocPath + ".md5"), - Artifact.javadocSha1 -> (javadocPath + ".sha1"), - Artifact.javadoc -> javadocPath, - Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), - Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), - Artifact.javadocSig -> (javadocPath + ".asc") + .copy( + extra = artifact.extra ++ Map( + "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? + .withDefaultChecksums + .withDefaultSignature, + "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above + .withDefaultChecksums + .withDefaultSignature )) } else artifact @@ -223,10 +213,8 @@ abstract class BaseMavenRepository( Artifact( path.mkString("/"), - Map( - Artifact.md5 -> "", - Artifact.sha1 -> "" - ), + Map.empty, + Map.empty, Attributes("pom", "") ) .withDefaultSignature @@ -246,6 +234,7 @@ abstract class BaseMavenRepository( Artifact( path.mkString("/"), Map.empty, + Map.empty, Attributes("pom", "") ) .withDefaultChecksums From 77fc66aae828a19950f34e52d4e3eba5b63b2736 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:43 +0100 Subject: [PATCH 09/13] Configurable concurrent download count (default: 6) --- .../main/scala/coursier/cli/Coursier.scala | 51 ++++++++++++------- files/src/main/scala/coursier/Files.scala | 13 ++++- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 1fc1357cc..b76a62e21 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -13,11 +13,15 @@ case class Coursier( scope: List[String], keepOptional: Boolean, fetch: Boolean, + @ExtraName("J") default: Boolean, + @ExtraName("S") sources: Boolean, + @ExtraName("D") javadoc: Boolean, @ExtraName("P") @ExtraName("cp") classpath: Boolean, @ExtraName("c") offline: Boolean, @ExtraName("v") verbose: List[Unit], @ExtraName("N") maxIterations: Int = 100, - @ExtraName("r") repository: List[String] + @ExtraName("r") repository: List[String], + @ExtraName("n") parallel: Option[Int] ) extends App { val verbose0 = verbose.length @@ -165,6 +169,8 @@ case class Coursier( print.flatMap(_ => fetchQuiet(modVers)) } + println(s"Resolving\n" + moduleVersions.map{case (mod, ver) => s" $mod:$ver"}.mkString("\n")) + val res = startRes .process .run(fetch0, maxIterations) @@ -205,11 +211,29 @@ case class Coursier( if (fetch || classpath) { println("") - val artifacts = res.artifacts + val artifacts0 = res.artifacts + val default0 = default || (!sources && !javadoc) + val artifacts = artifacts0 + .flatMap{ artifact => + var l = List.empty[Artifact] + if (sources) + l = artifact.extra.get("sources").toList ::: l + if (javadoc) + l = artifact.extra.get("javadoc").toList ::: l + if (default0) + l = artifact :: l - val files = cache - .files() - .copy(logger = Some(logger)) + l + } + + val files = { + var files0 = cache + .files() + .copy(logger = Some(logger)) + for (n <- parallel) + files0 = files0.copy(concurrentDownloadCount = n) + files0 + } val tasks = artifacts.map(artifact => files.file(artifact, cachePolicy).run.map(artifact.->)) val task = Task.gatherUnordered(tasks) @@ -225,18 +249,11 @@ case class Coursier( } } - if (classpath) - Console.out.println( - files0 - .map(_.toString) - .mkString(File.pathSeparator) - ) - else - println( - files0 - .map(_.toString) - .mkString("\n") - ) + Console.out.println( + files0 + .map(_.toString) + .mkString(if (classpath) File.pathSeparator else "\n") + ) } } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index 46b337a9b..8a6613c72 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -1,22 +1,29 @@ package coursier import java.net.{ URI, URL } +import java.util.concurrent.{ Executors, ExecutorService } import scala.annotation.tailrec import scalaz.{ -\/, \/-, \/, EitherT } -import scalaz.concurrent.Task +import scalaz.concurrent.{ Task, Strategy } import java.io._ case class Files( cache: Seq[(String, File)], tmp: () => File, - logger: Option[Files.Logger] = None + logger: Option[Files.Logger] = None, + concurrentDownloadCount: Int = Files.defaultConcurrentDownloadCount ) { + lazy val defaultPool = + Executors.newFixedThreadPool(concurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) + def file( artifact: Artifact, cachePolicy: CachePolicy + )(implicit + pool: ExecutorService = defaultPool ): EitherT[Task, String, File] = { if (artifact.url.startsWith("file:///")) { @@ -94,6 +101,8 @@ case class Files( } object Files { + + val defaultConcurrentDownloadCount = 6 // FIXME This kind of side-effecting API is lame, we should aim at a more functional one. trait Logger { From 9ccf972f8a44043069ca7f6014b6ea4585eda08d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:43 +0100 Subject: [PATCH 10/13] Force download option --- cli/src/main/scala/coursier/cli/Coursier.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index b76a62e21..448518668 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -18,6 +18,7 @@ case class Coursier( @ExtraName("D") javadoc: Boolean, @ExtraName("P") @ExtraName("cp") classpath: Boolean, @ExtraName("c") offline: Boolean, + @ExtraName("f") force: Boolean, @ExtraName("v") verbose: List[Unit], @ExtraName("N") maxIterations: Int = 100, @ExtraName("r") repository: List[String], @@ -36,6 +37,12 @@ case class Coursier( def println(s: String) = Console.err.println(s) + if (force && offline) { + println("Error: --offline (-c) and --force (-f) options can't be specified at the same time.") + sys.exit(255) + } + + def defaultLogger: MavenRepository.Logger with Files.Logger = new MavenRepository.Logger with Files.Logger { def downloading(url: String) = @@ -85,6 +92,8 @@ case class Coursier( implicit val cachePolicy = if (offline) CachePolicy.LocalOnly + else if (force) + CachePolicy.ForceDownload else CachePolicy.Default From 4be4b0f38104bd8455d69289be7c5ffc81711b01 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:44 +0100 Subject: [PATCH 11/13] Quiet option --- .../main/scala/coursier/cli/Coursier.scala | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 448518668..8226dd6c1 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -19,13 +19,17 @@ case class Coursier( @ExtraName("P") @ExtraName("cp") classpath: Boolean, @ExtraName("c") offline: Boolean, @ExtraName("f") force: Boolean, + @ExtraName("q") quiet: Boolean, @ExtraName("v") verbose: List[Unit], @ExtraName("N") maxIterations: Int = 100, @ExtraName("r") repository: List[String], @ExtraName("n") parallel: Option[Int] ) extends App { - val verbose0 = verbose.length + val verbose0 = { + verbose.length + + (if (quiet) 1 else 0) + } val scopes0 = if (scope.isEmpty) List(Scope.Compile, Scope.Runtime) @@ -87,7 +91,13 @@ case class Coursier( ) } - val logger = if (verbose0 <= 0) defaultLogger else verboseLogger + val logger = + if (verbose0 < 0) + None + else if (verbose0 == 0) + Some(defaultLogger) + else + Some(verboseLogger) implicit val cachePolicy = if (offline) @@ -135,7 +145,7 @@ case class Coursier( .unzip val repositories = repositories0 - .map(_.copy(logger = Some(logger))) + .map(_.copy(logger = logger)) val (splitDependencies, malformed) = remainingArgs.toList @@ -178,7 +188,8 @@ case class Coursier( print.flatMap(_ => fetchQuiet(modVers)) } - println(s"Resolving\n" + moduleVersions.map{case (mod, ver) => s" $mod:$ver"}.mkString("\n")) + if (verbose0 >= 0) + println(s"Resolving\n" + moduleVersions.map{case (mod, ver) => s" $mod:$ver"}.mkString("\n")) val res = startRes .process @@ -202,7 +213,8 @@ case class Coursier( val trDeps = res.minDependencies.toList.sortBy(repr) - println("\n" + trDeps.map(repr).distinct.mkString("\n")) + if (verbose0 >= 0) + println("\n" + trDeps.map(repr).distinct.mkString("\n")) if (res.conflicts.nonEmpty) { // Needs test @@ -238,7 +250,7 @@ case class Coursier( val files = { var files0 = cache .files() - .copy(logger = Some(logger)) + .copy(logger = logger) for (n <- parallel) files0 = files0.copy(concurrentDownloadCount = n) files0 From c061717521139af336c1e28c6493342b6d2cfa03 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:45 +0100 Subject: [PATCH 12/13] Printing artifact count --- cli/src/main/scala/coursier/cli/Coursier.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 8226dd6c1..e983a621f 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -257,7 +257,11 @@ case class Coursier( } val tasks = artifacts.map(artifact => files.file(artifact, cachePolicy).run.map(artifact.->)) - val task = Task.gatherUnordered(tasks) + def printTask = Task{ + if (verbose0 >= 0 && artifacts.nonEmpty) + println(s"Found ${artifacts.length} artifacts") + } + val task = printTask.flatMap(_ => Task.gatherUnordered(tasks)) val results = task.run val errors = results.collect{case (artifact, -\/(err)) => artifact -> err } From a35e30c8ad48ff333e49c3b0bc829ecacdbe6200 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 16:30:33 +0200 Subject: [PATCH 13/13] Scala 2.10 fix --- files/src/main/scala/coursier/Cache.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/files/src/main/scala/coursier/Cache.scala b/files/src/main/scala/coursier/Cache.scala index ebd80eff9..8de434f51 100644 --- a/files/src/main/scala/coursier/Cache.scala +++ b/files/src/main/scala/coursier/Cache.scala @@ -13,12 +13,14 @@ object Cache { lines .find(_.startsWith("maven:")) .map(_.stripPrefix("maven:").trim) + .toSeq .contains("true") def ivyLike = lines .find(_.startsWith("ivy-like:")) .map(_.stripPrefix("ivy-like:").trim) + .toSeq .contains("true") def base =