From 9fd56848a8c2978a6b71325df5dc50a53b214f18 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 25 Jun 2015 00:18:57 +0100 Subject: [PATCH] Refactor artifact API --- .../main/scala/coursier/cli/Coursier.scala | 76 ++---- .../coursier/core/DefaultFetchMetadata.scala | 94 +++++++ .../src/main/scala/coursier/core/Remote.scala | 138 ---------- .../coursier/core/compatibility/package.scala | 4 + .../scala/coursier/repository/package.scala | 11 +- .../test/scala/coursier/test/JsTests.scala | 5 +- .../coursier/core/DefaultFetchMetadata.scala | 101 ++++++++ .../src/main/scala/coursier/core/Remote.scala | 243 ------------------ .../coursier/core/compatibility/package.scala | 2 + .../scala/coursier/repository/package.scala | 11 +- .../coursier/test/compatibility/package.scala | 4 +- .../scala/coursier/core/Definitions.scala | 25 ++ .../main/scala/coursier/core/Repository.scala | 196 ++++++++++++-- .../main/scala/coursier/core/Resolution.scala | 61 +++-- core/src/main/scala/coursier/package.scala | 18 +- .../scala/coursier/test/CentralTests.scala | 10 +- .../scala/coursier/test/ResolutionTests.scala | 32 +-- .../scala/coursier/test/TestRepository.scala | 5 +- files/src/main/scala/coursier/Files.scala | 121 +++++++++ project/Coursier.scala | 14 +- web/src/main/scala/coursier/web/Backend.scala | 32 +-- 21 files changed, 665 insertions(+), 538 deletions(-) create mode 100644 core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala delete mode 100644 core-js/src/main/scala/coursier/core/Remote.scala create mode 100644 core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala delete mode 100644 core-jvm/src/main/scala/coursier/core/Remote.scala create mode 100644 files/src/main/scala/coursier/Files.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index a131660e2..3f06b9891 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -5,10 +5,9 @@ import java.io.File import caseapp._ import coursier.core.{CachePolicy, Parse} -import coursier.core.{ArtifactDownloaderLogger, RemoteLogger, ArtifactDownloader} +import coursier.core.MetadataFetchLogger import scalaz.concurrent.Task -import scalaz.{-\/, \/-} case class Coursier(scope: List[String], keepOptional: Boolean, @@ -20,13 +19,14 @@ case class Coursier(scope: List[String], else scope.map(Parse.scope) val scopes = scopes0.toSet - val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/central") + val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/metadata/central") + val centralFilesCacheDir = new File(sys.props("user.home") + "/.coursier/cache/files/central") val base = centralCacheDir.toURI def fileRepr(f: File) = base.relativize(f.toURI).getPath - val logger: RemoteLogger with ArtifactDownloaderLogger = new RemoteLogger with ArtifactDownloaderLogger { + val logger: MetadataFetchLogger with FilesLogger = new MetadataFetchLogger with FilesLogger { def println(s: String) = Console.err.println(s) def downloading(url: String) = @@ -53,35 +53,36 @@ case class Coursier(scope: List[String], ) } - val cachedMavenCentral = repository.mavenCentral.copy(cache = Some(centralCacheDir), logger = Some(logger)) + val cachedMavenCentral = repository.mavenCentral.copy( + fetchMetadata = repository.mavenCentral.fetchMetadata.copy( + cache = Some(centralCacheDir), + logger = Some(logger) + ) + ) val repositories = Seq[Repository]( cachedMavenCentral ) - lazy val downloaders = Map[Repository, ArtifactDownloader]( - cachedMavenCentral -> ArtifactDownloader(repository.mavenCentral.root, centralCacheDir, logger = Some(logger)) - ) - - val (splitArtifacts, malformed) = remainingArgs.toList + val (splitDependencies, malformed) = remainingArgs.toList .map(_.split(":", 3).toSeq) .partition(_.length == 3) - if (splitArtifacts.isEmpty) { - Console.err.println("Usage: coursier artifacts...") + if (splitDependencies.isEmpty) { + Console.err.println("Usage: coursier dependencies...") sys exit 1 } if (malformed.nonEmpty) { - Console.err.println(s"Malformed artifacts:\n${malformed.map(_.mkString(":")).mkString("\n")}") + Console.err.println(s"Malformed dependencies:\n${malformed.map(_.mkString(":")).mkString("\n")}") sys exit 1 } - val modules = splitArtifacts.map{ + val moduleVersions = splitDependencies.map{ case Seq(org, name, version) => (Module(org, name), version) } - val deps = modules.map{case (mod, ver) => + val deps = moduleVersions.map{case (mod, ver) => Dependency(mod, ver, scope = Scope.Runtime) } @@ -99,7 +100,7 @@ case class Coursier(scope: List[String], def repr(dep: Dependency) = { // dep.version can be an interval, whereas the one from project can't - val version = res.projectsCache.get(dep.moduleVersion).map(_._2.version).getOrElse(dep.version) + val version = res.projectCache.get(dep.moduleVersion).map(_._2.version).getOrElse(dep.version) val extra = if (version == dep.version) "" else s" ($version for ${dep.version})" @@ -116,11 +117,11 @@ case class Coursier(scope: List[String], println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") } - val errDeps = trDeps.filter(dep => res.errors.contains(dep.moduleVersion)) - if (errDeps.nonEmpty) { - println(s"${errDeps.size} error(s):") - for (dep <- errDeps) { - println(s" ${dep.module}:\n ${res.errors(dep.moduleVersion).mkString("\n").replace("\n", " \n")}") + val errors = res.errors + if (errors.nonEmpty) { + println(s"${errors.size} error(s):") + for ((dep, errs) <- errors) { + println(s" ${dep.module}:\n ${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") } } @@ -129,38 +130,11 @@ case class Coursier(scope: List[String], val cachePolicy: CachePolicy = CachePolicy.Default - val m = res.minDependencies.groupBy(dep => res.projectsCache.get(dep.moduleVersion).map(_._1)) - val (notFound, remaining0) = m.partition(_._1.isEmpty) - if (notFound.nonEmpty) { - val notFound0 = notFound.values.flatten.toList.map(repr).sorted - println(s"Not found:${notFound0.mkString("\n")}") - } + val artifacts = res.artifacts - val (remaining, downloaderNotFound) = remaining0.partition(t => downloaders.contains(t._1.get)) - if (downloaderNotFound.nonEmpty) { - val downloaderNotFound0 = downloaderNotFound.values.flatten.toList.map(repr).sorted - println(s"Don't know how to download:${downloaderNotFound0.mkString("\n")}") - } - - val sorted = remaining - .toList - .map{ case (Some(repo), deps) => repo -> deps.toList.sortBy(repr) } - .sortBy(_._1.toString) // ... - - val tasks = - for { - (repo, deps) <- sorted - dl = downloaders(repo) - dep <- deps - (_, proj) = res.projectsCache(dep.moduleVersion) - } yield { - dl.artifacts(dep, proj, cachePolicy = cachePolicy).map { results => - val errorCount = results.count{case -\/(_) => true; case _ => false} - val resultsRepr = results.map(_.map(fileRepr).merge).map(" " + _).mkString("\n") - println(s"${repr(dep)} (${results.length} artifact(s)${if (errorCount > 0) s", $errorCount error(s)" else ""}):\n$resultsRepr") - } - } + val files = new Files(Seq(cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir), () => ???, Some(logger)) + val tasks = artifacts.map(files.file(_, cachePolicy).run) val task = Task.gatherUnordered(tasks) task.run diff --git a/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala new file mode 100644 index 000000000..d41b995bf --- /dev/null +++ b/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala @@ -0,0 +1,94 @@ +package coursier +package core + +import org.scalajs.dom.raw.{Event, XMLHttpRequest} + +import scala.concurrent.{ExecutionContext, Promise, Future} +import scalaz.{-\/, \/-, EitherT} +import scalaz.concurrent.Task + +import scala.scalajs.js +import js.Dynamic.{global => g} + +import scala.scalajs.js.timers._ + +object DefaultFetchMetadata { + + def encodeURIComponent(s: String): String = + g.encodeURIComponent(s).asInstanceOf[String] + + lazy val jsonpAvailable = !js.isUndefined(g.jsonp) + + /** Available if we're running on node, and package xhr2 is installed */ + lazy val xhr = g.require("xhr2") + def xhrReq() = + js.Dynamic.newInstance(xhr)().asInstanceOf[XMLHttpRequest] + + def fetchTimeout(target: String, p: Promise[_]) = + setTimeout(5000) { + if (!p.isCompleted) { + p.failure(new Exception(s"Timeout when fetching $target")) + } + } + + // FIXME Take into account HTTP error codes from YQL response + def proxiedJsonp(url: String)(implicit executionContext: ExecutionContext): Future[String] = { + val url0 = + "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20xml%20where%20url%3D%22" + + encodeURIComponent(url) + + "%22&format=jsonp&diagnostics=true" + + val p = Promise[String]() + + g.jsonp(url0, (res: js.Dynamic) => if (!p.isCompleted) { + val success = !js.isUndefined(res) && !js.isUndefined(res.results) + if (success) + p.success(res.results.asInstanceOf[js.Array[String]].mkString("\n")) + else + p.failure(new Exception(s"Fetching $url ($url0)")) + }) + + fetchTimeout(s"$url ($url0)", p) + p.future + } + + def get(url: String)(implicit executionContext: ExecutionContext): Future[String] = + if (jsonpAvailable) + proxiedJsonp(url) + else { + val p = Promise[String]() + val xhrReq0 = xhrReq() + val f = { _: Event => + p.success(xhrReq0.responseText) + } + xhrReq0.onload = f + + xhrReq0.open("GET", url) + xhrReq0.send() + + fetchTimeout(url, p) + p.future + } + +} + +trait Logger { + def fetching(url: String): Unit + def fetched(url: String): Unit + def other(url: String, msg: String): Unit +} + +case class DefaultFetchMetadata(root: String, + logger: Option[Logger] = None) extends FetchMetadata { + def apply(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, String] = { + + EitherT( + Task { implicit ec => + DefaultFetchMetadata.get(root + artifact.url) + .map(\/-(_)) + .recover{case e: Exception => -\/(e.getMessage)} + } + ) + } +} \ No newline at end of file diff --git a/core-js/src/main/scala/coursier/core/Remote.scala b/core-js/src/main/scala/coursier/core/Remote.scala deleted file mode 100644 index 6aa2d2700..000000000 --- a/core-js/src/main/scala/coursier/core/Remote.scala +++ /dev/null @@ -1,138 +0,0 @@ -package coursier -package core - -import org.scalajs.dom.raw.{Event, XMLHttpRequest} - -import scala.concurrent.{ExecutionContext, Promise, Future} -import scalaz.{-\/, \/-, \/, EitherT} -import scalaz.concurrent.Task - -import scala.scalajs.js -import js.Dynamic.{global => g} - -import scala.scalajs.js.timers._ - -object Remote { - - def encodeURIComponent(s: String): String = - g.encodeURIComponent(s).asInstanceOf[String] - - lazy val jsonpAvailable = !js.isUndefined(g.jsonp) - - /** Available if we're running on node, and package xhr2 is installed */ - lazy val xhr = g.require("xhr2") - def xhrReq() = - js.Dynamic.newInstance(xhr)().asInstanceOf[XMLHttpRequest] - - def fetchTimeout(target: String, p: Promise[_]) = - setTimeout(5000) { - if (!p.isCompleted) { - p.failure(new Exception(s"Timeout when fetching $target")) - } - } - - def proxiedJsonp(url: String)(implicit executionContext: ExecutionContext): Future[String] = { - val url0 = - "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20xml%20where%20url%3D%22" + - encodeURIComponent(url) + - "%22&format=jsonp&diagnostics=true" - - val p = Promise[String]() - - g.jsonp(url0, (res: js.Dynamic) => if (!p.isCompleted) { - val success = !js.isUndefined(res) && !js.isUndefined(res.results) - if (success) - p.success(res.results.asInstanceOf[js.Array[String]].mkString("\n")) - else - p.failure(new Exception(s"Fetching $url ($url0)")) - }) - - fetchTimeout(s"$url ($url0)", p) - p.future - } - - def get(url: String)(implicit executionContext: ExecutionContext): Future[Either[String, Xml.Node]] = - if (jsonpAvailable) - proxiedJsonp(url).map(compatibility.xmlParse) - else { - val p = Promise[Either[String, Xml.Node]]() - val xhrReq0 = xhrReq() - val f = { _: Event => - p.success(compatibility.xmlParse(xhrReq0.responseText)) - } - xhrReq0.onload = f - - xhrReq0.open("GET", url) - xhrReq0.send() - - fetchTimeout(url, p) - p.future - } - -} - -trait Logger { - def fetching(url: String): Unit - def fetched(url: String): Unit - def other(url: String, msg: String): Unit -} - -case class Remote(base: String, logger: Option[Logger] = None) extends MavenRepository { - - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { - - val path = { - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version.pom" - ) - } .map(Remote.encodeURIComponent) - - val url = base + path.mkString("/") - - EitherT(Task{ implicit ec => - logger.foreach(_.fetching(url)) - Remote.get(url).recover{case e: Exception => Left(e.getMessage)}.map{ eitherXml => - logger.foreach(_.fetched(url)) - for { - xml <- \/.fromEither(eitherXml) - _ = logger.foreach(_.other(url, "is XML")) - _ <- if (xml.label == "project") \/-(()) else -\/(s"Project definition not found (got '${xml.label}')") - _ = logger.foreach(_.other(url, "project definition found")) - proj <- Xml.project(xml) - _ = logger.foreach(_.other(url, "project definition ok")) - } yield proj - } - }) - } - - def versions(organization: String, - name: String, - cachePolicy: CachePolicy): EitherT[Task, String, Versions] = { - - val path = { - organization.split('.').toSeq ++ Seq( - name, - "maven-metadata.xml" - ) - } .map(Remote.encodeURIComponent) - - val url = base + path.mkString("/") - - EitherT(Task{ implicit ec => - logger.foreach(_.fetching(url)) - Remote.get(url).recover{case e: Exception => Left(e.getMessage)}.map{ eitherXml => - logger.foreach(_.fetched(url)) - for { - xml <- \/.fromEither(eitherXml) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - versions <- Xml.versions(xml) - } yield versions - } - }) - } - -} diff --git a/core-js/src/main/scala/coursier/core/compatibility/package.scala b/core-js/src/main/scala/coursier/core/compatibility/package.scala index 4c57573d7..5c940cb9d 100644 --- a/core-js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core-js/src/main/scala/coursier/core/compatibility/package.scala @@ -1,6 +1,7 @@ package coursier.core import scala.scalajs.js +import js.Dynamic.{global => g} import org.scalajs.dom.raw.NodeList package object compatibility { @@ -94,4 +95,7 @@ package object compatibility { Right(doc.fold(Xml.Node.empty)(fromNode)) } + def encodeURIComponent(s: String): String = + g.encodeURIComponent(s).asInstanceOf[String] + } diff --git a/core-js/src/main/scala/coursier/repository/package.scala b/core-js/src/main/scala/coursier/repository/package.scala index 59d758a39..c448f25c5 100644 --- a/core-js/src/main/scala/coursier/repository/package.scala +++ b/core-js/src/main/scala/coursier/repository/package.scala @@ -1,13 +1,12 @@ package coursier +import coursier.core.DefaultFetchMetadata + package object repository { - type Remote = core.Remote - val Remote: core.Remote.type = core.Remote + val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/")) - val mavenCentral = Remote("https://repo1.maven.org/maven2/") - - val sonatypeReleases = Remote("https://oss.sonatype.org/content/repositories/releases/") - val sonatypeSnapshots = Remote("https://oss.sonatype.org/content/repositories/snapshots/") + val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) + val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) } diff --git a/core-js/src/test/scala/coursier/test/JsTests.scala b/core-js/src/test/scala/coursier/test/JsTests.scala index 5888d9331..9738b2600 100644 --- a/core-js/src/test/scala/coursier/test/JsTests.scala +++ b/core-js/src/test/scala/coursier/test/JsTests.scala @@ -1,7 +1,7 @@ package coursier package test -import coursier.core.Remote +import coursier.core.DefaultFetchMetadata import coursier.test.compatibility._ import utest._ @@ -18,7 +18,8 @@ object JsTests extends TestSuite { } 'get{ - Remote.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + DefaultFetchMetadata.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + .map(core.compatibility.xmlParse) .map{ xml => assert(xml.right.toOption.exists(_.label == "project")) } diff --git a/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala new file mode 100644 index 000000000..30f7f87af --- /dev/null +++ b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala @@ -0,0 +1,101 @@ +package coursier +package core + +import java.io._ +import java.net.URL + +import scala.io.Codec +import scalaz._, Scalaz._ +import scalaz.concurrent.Task + +trait MetadataFetchLogger { + def downloading(url: String): Unit + def downloaded(url: String, success: Boolean): Unit + def readingFromCache(f: File): Unit + def puttingInCache(f: File): Unit +} + +case class DefaultFetchMetadata(root: String, + cache: Option[File] = None, + logger: Option[MetadataFetchLogger] = None) extends FetchMetadata { + + def apply(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, String] = { + lazy val localFile = { + for { + cache0 <- cache.toRightDisjunction("No cache") + f = new File(cache0, artifact.url) + } yield f + } + + def locally = { + Task { + for { + f0 <- localFile + f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") + content <- \/.fromTryCatchNonFatal{ + logger.foreach(_.readingFromCache(f)) + scala.io.Source.fromFile(f)(Codec.UTF8).mkString + }.leftMap(_.getMessage) + } yield content + } + } + + def remote = { + val urlStr = root + artifact.url + val url = new URL(urlStr) + + def log = Task(logger.foreach(_.downloading(urlStr))) + def get = DefaultFetchMetadata.readFully(url.openStream()) + + log.flatMap(_ => get) + } + + def save(s: String) = { + localFile.fold(_ => Task.now(()), f => + Task { + if (!f.exists()) { + logger.foreach(_.puttingInCache(f)) + f.getParentFile.mkdirs() + val w = new PrintWriter(f) + try w.write(s) + finally w.close() + () + } + } + ) + } + + EitherT(cachePolicy.saving(locally)(remote)(save)) + } + +} + +object DefaultFetchMetadata { + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap(_.getMessage) + } + +} diff --git a/core-jvm/src/main/scala/coursier/core/Remote.scala b/core-jvm/src/main/scala/coursier/core/Remote.scala deleted file mode 100644 index e584e9347..000000000 --- a/core-jvm/src/main/scala/coursier/core/Remote.scala +++ /dev/null @@ -1,243 +0,0 @@ -package coursier -package core - -import java.io._ -import java.net.URL - -import scala.annotation.tailrec -import scala.io.Codec -import scalaz._, Scalaz._ -import scalaz.concurrent.Task - -// FIXME This kind of side-effecting API is lame, we should aim at a more functional one. -trait ArtifactDownloaderLogger { - def foundLocally(f: File): Unit - def downloadingArtifact(url: String): Unit - def downloadedArtifact(url: String, success: Boolean): Unit -} - -case class ArtifactDownloader(root: String, cache: File, logger: Option[ArtifactDownloaderLogger] = None) { - var bufferSize = 1024*1024 - - def artifact(module: Module, - version: String, - attributes: Attributes, - cachePolicy: CachePolicy): EitherT[Task, String, File] = { - - val relPath = - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version${Some(attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${attributes.`type`}" - ) - - val file = (cache /: relPath)(new File(_, _)) - - def locally = { - Task { - if (file.exists()) { - logger.foreach(_.foundLocally(file)) - \/-(file) - } - else -\/("Not found in cache") - } - } - - def remote = { - // FIXME A lot of things can go wrong here and are not properly handled: - // - checksums should be validated - // - what if the connection gets closed during the transfer (partial file on disk)? - // - what if someone is trying to write this file at the same time? (no locking of any kind yet) - // - ... - - val urlStr = root + relPath.mkString("/") - - Task { - try { - file.getParentFile.mkdirs() - - logger.foreach(_.downloadingArtifact(urlStr)) - - val url = new URL(urlStr) - val b = Array.fill[Byte](bufferSize)(0) - val in = new BufferedInputStream(url.openStream(), bufferSize) - - try { - val out = new FileOutputStream(file) - try { - @tailrec - def helper(): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - helper() - } - } - - helper() - } finally out.close() - } finally in.close() - - logger.foreach(_.downloadedArtifact(urlStr, success = true)) - \/-(file) - } - catch { case e: Exception => - logger.foreach(_.downloadedArtifact(urlStr, success = false)) - -\/(e.getMessage) - } - } - } - - EitherT(cachePolicy(locally)(remote)) - } - - def artifacts(dependency: Dependency, - project: Project, - cachePolicy: CachePolicy = CachePolicy.Default): Task[Seq[String \/ File]] = { - - // Important: using version from project, as the one from dependency can be an interval - artifact(dependency.module, project.version, dependency.attributes, cachePolicy = cachePolicy).run.map(Seq(_)) - } - -} - -// FIXME Comment of ArtifactDownloaderLogger applies here too -trait RemoteLogger { - def downloading(url: String): Unit - def downloaded(url: String, success: Boolean): Unit - def readingFromCache(f: File): Unit - def puttingInCache(f: File): Unit -} - -object Remote { - - def readFullySync(is: InputStream) = { - val buffer = new ByteArrayOutputStream() - val data = Array.ofDim[Byte](16384) - - var nRead = is.read(data, 0, data.length) - while (nRead != -1) { - buffer.write(data, 0, nRead) - nRead = is.read(data, 0, data.length) - } - - buffer.flush() - buffer.toByteArray - } - - def readFully(is: => InputStream) = - Task { - \/.fromTryCatchNonFatal { - val is0 = is - val b = - try readFullySync(is0) - finally is0.close() - - new String(b, "UTF-8") - } .leftMap(_.getMessage) - } - -} - -case class Remote(root: String, - cache: Option[File] = None, - logger: Option[RemoteLogger] = None) extends MavenRepository { - - private def get(path: Seq[String], - cachePolicy: CachePolicy): EitherT[Task, String, String] = { - - lazy val localFile = { - for { - cache0 <- cache.toRightDisjunction("No cache") - f = (cache0 /: path)(new File(_, _)) - } yield f - } - - def locally = { - Task { - for { - f0 <- localFile - f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") - content <- \/.fromTryCatchNonFatal{ - logger.foreach(_.readingFromCache(f)) - scala.io.Source.fromFile(f)(Codec.UTF8).mkString - }.leftMap(_.getMessage) - } yield content - } - } - - def remote = { - val urlStr = root + path.mkString("/") - val url = new URL(urlStr) - - def log = Task(logger.foreach(_.downloading(urlStr))) - def get = Remote.readFully(url.openStream()) - - log.flatMap(_ => get) - } - - def save(s: String) = { - localFile.fold(_ => Task.now(()), f => - Task { - if (!f.exists()) { - logger.foreach(_.puttingInCache(f)) - f.getParentFile.mkdirs() - val w = new PrintWriter(f) - try w.write(s) - finally w.close() - () - } - } - ) - } - - EitherT(cachePolicy.saving(locally)(remote)(save)) - } - - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { - - val path = - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version.pom" - ) - - val task = get(path, cachePolicy).run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") - proj <- Xml.project(xml) - } yield proj - ) - - EitherT(task) - } - - def versions(organization: String, - name: String, - cachePolicy: CachePolicy): EitherT[Task, String, Versions] = { - - val path = - organization.split('.').toSeq ++ Seq( - name, - "maven-metadata.xml" - ) - - val task = get(path, cachePolicy).run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - versions <- Xml.versions(xml) - } yield versions - ) - - EitherT(task) - } -} diff --git a/core-jvm/src/main/scala/coursier/core/compatibility/package.scala b/core-jvm/src/main/scala/coursier/core/compatibility/package.scala index 5eb8cbc97..eb4cb154c 100644 --- a/core-jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core-jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -25,4 +25,6 @@ package object compatibility { .map(fromNode) } + def encodeURIComponent(s: String): String = + new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString } diff --git a/core-jvm/src/main/scala/coursier/repository/package.scala b/core-jvm/src/main/scala/coursier/repository/package.scala index 59d758a39..c448f25c5 100644 --- a/core-jvm/src/main/scala/coursier/repository/package.scala +++ b/core-jvm/src/main/scala/coursier/repository/package.scala @@ -1,13 +1,12 @@ package coursier +import coursier.core.DefaultFetchMetadata + package object repository { - type Remote = core.Remote - val Remote: core.Remote.type = core.Remote + val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/")) - val mavenCentral = Remote("https://repo1.maven.org/maven2/") - - val sonatypeReleases = Remote("https://oss.sonatype.org/content/repositories/releases/") - val sonatypeSnapshots = Remote("https://oss.sonatype.org/content/repositories/snapshots/") + val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) + val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) } diff --git a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala index 250a46515..10c93544c 100644 --- a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala +++ b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.core.Remote +import coursier.core.DefaultFetchMetadata import scala.concurrent.{ExecutionContext, Future} import scalaz.concurrent.Task @@ -17,7 +17,7 @@ package object compatibility { def is = getClass.getClassLoader .getResource(path).openStream() - new String(Remote.readFullySync(is), "UTF-8") + new String(DefaultFetchMetadata.readFullySync(is), "UTF-8") } } diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index 6a331e291..75c775633 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -70,6 +70,7 @@ case class Profile(id: String, dependencyManagement: Seq[Dependency], properties: Map[String, String]) +// FIXME Move to MavenRepository? case class Versions(latest: String, release: String, available: List[String], @@ -78,3 +79,27 @@ case class Versions(latest: String, object Versions { case class DateTime(year: Int, month: Int, day: Int, hour: Int, minute: Int, second: Int) } + +case class Artifact(url: String, + extra: Map[String, String], + attributes: Attributes) + +object Artifact { + val md5 = "md5" + val sha1 = "sha1" + val sig = "pgp" + val sigMd5 = "md5-pgp" + val sigSha1 = "sha1-pgp" + val sources = "src" + val sourcesMd5 = "md5-src" + val sourcesSha1 = "sha1-src" + val sourcesSig = "src-pgp" + val sourcesSigMd5 = "md5-src-pgp" + val sourcesSigSha1 = "sha1-src-pgp" + val javadoc = "javadoc" + val javadocMd5 = "md5-javadoc" + val javadocSha1 = "sha1-javadoc" + val javadocSig = "javadoc-pgp" + val javadocSigMd5 = "md5-javadoc-pgp" + val javadocSigSha1 = "sha1-javadoc-pgp" +} diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index c953d64cc..d95d32eef 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -3,9 +3,11 @@ package coursier.core import scalaz.{-\/, \/-, \/, EitherT} import scalaz.concurrent.Task +import coursier.core.compatibility.encodeURIComponent + trait Repository { def find(module: Module, version: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Project] - def versions(organization: String, name: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Versions] + def artifacts(dependency: Dependency, project: Project): Seq[Artifact] } sealed trait CachePolicy { @@ -37,7 +39,132 @@ object CachePolicy { } } -trait MavenRepository extends Repository { +object Repository { + implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { + def withDefaultChecksums: Artifact = + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.md5 -> (underlying.url + ".md5"), + Artifact.sha1 -> (underlying.url + ".sha1") + )) + def withDefaultSignature: Artifact = + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.sigMd5 -> (underlying.url + ".asc.md5"), + Artifact.sigSha1 -> (underlying.url + ".asc.sha1"), + Artifact.sig -> (underlying.url + ".asc") + )) + def withJavadocSources: Artifact = { + val base = underlying.url.stripSuffix(".jar") + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.sourcesMd5 -> (base + "-sources.jar.md5"), + Artifact.sourcesSha1 -> (base + "-sources.jar.sha1"), + Artifact.sources -> (base + "-sources.jar"), + Artifact.sourcesSigMd5 -> (base + "-sources.jar.asc.md5"), + Artifact.sourcesSigSha1 -> (base + "-sources.jar.asc.sha1"), + Artifact.sourcesSig -> (base + "-sources.jar.asc"), + Artifact.javadocMd5 -> (base + "-javadoc.jar.md5"), + Artifact.javadocSha1 -> (base + "-javadoc.jar.sha1"), + Artifact.javadoc -> (base + "-javadoc.jar"), + Artifact.javadocSigMd5 -> (base + "-javadoc.jar.asc.md5"), + Artifact.javadocSigSha1 -> (base + "-javadoc.jar.asc.sha1"), + Artifact.javadocSig -> (base + "-javadoc.asc.jar") + )) + } + } +} + +trait FetchMetadata { + def root: String + def apply(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, String] +} + +case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, + ivyLike: Boolean = false) extends Repository { + + import Repository._ + + def projectArtifact(module: Module, version: String): Artifact = { + if (ivyLike) ??? + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + version, + s"${module.name}-$version.pom" + ) + ) .map(encodeURIComponent) + + Artifact( + path.mkString("/"), + Map( + Artifact.md5 -> "", + Artifact.sha1 -> "" + ), + Attributes("pom", "") + ) + .withDefaultSignature + } + } + + def versionsArtifact(module: Module): Option[Artifact] = + if (ivyLike) None + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + "maven-metadata.xml" + ) + ) .map(encodeURIComponent) + + val artifact = + Artifact( + path.mkString("/"), + Map.empty, + Attributes("pom", "") + ) + .withDefaultChecksums + + Some(artifact) + } + + def versions(module: Module, + cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Versions] = { + + EitherT( + versionsArtifact(module) match { + case None => Task.now(-\/("Not supported")) + case Some(artifact) => + fetchMetadata(artifact, cachePolicy) + .run + .map(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") + versions <- Xml.versions(xml) + } yield versions + ) + } + ) + } + + def findNoInterval(module: Module, + version: String, + cachePolicy: CachePolicy): EitherT[Task, String, Project] = { + + EitherT { + fetchMetadata(projectArtifact(module, version), cachePolicy) + .run + .map(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") + proj <- Xml.project(xml) + } yield proj + ) + } + } def find(module: Module, version: String, @@ -46,28 +173,55 @@ trait MavenRepository extends Repository { Parse.versionInterval(version).filter(_.isValid) match { case None => findNoInterval(module, version, cachePolicy) case Some(itv) => - versions(module.organization, module.name, cachePolicy).flatMap { versions0 => - val eitherVersion = { - val release = Version(versions0.release) - if (itv.contains(release)) \/-(versions0.release) - else { - val inInterval = versions0.available.map(Version(_)).filter(itv.contains) - if (inInterval.isEmpty) -\/(s"No version found for $version") - else \/-(inInterval.max.repr) + versions(module, cachePolicy) + .flatMap { versions0 => + val eitherVersion = { + val release = Version(versions0.release) + + if (itv.contains(release)) \/-(versions0.release) + else { + val inInterval = versions0.available + .map(Version(_)) + .filter(itv.contains) + + if (inInterval.isEmpty) -\/(s"No version found for $version") + else \/-(inInterval.max.repr) + } + } + + eitherVersion match { + case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason))) + case \/-(version0) => + findNoInterval(module, version0, cachePolicy) + .map(_.copy(versions = Some(versions0))) } } - - eitherVersion match { - case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason))) - case \/-(version0) => findNoInterval(module, version0, cachePolicy) - .map(_.copy(versions = Some(versions0))) - } - } } } - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] + def artifacts(dependency: Dependency, + project: Project): Seq[Artifact] = { -} + val path = + dependency.module.organization.split('.').toSeq ++ Seq( + dependency.module.name, + project.version, + s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + ) + + var artifact = + Artifact( + fetchMetadata.root + path.mkString("/"), + Map.empty, + dependency.attributes + ) + .withDefaultChecksums + + if (dependency.attributes.`type` == "jar") + artifact = artifact + .withDefaultSignature + .withJavadocSources + + Seq(artifact) + } +} \ No newline at end of file diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index 8596bb480..1f771a258 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -322,14 +322,14 @@ object Resolution { * * @param dependencies: current set of dependencies * @param conflicts: conflicting dependencies - * @param projectsCache: cache of known projects - * @param errors: keeps track of the modules whose project definition could not be found + * @param projectCache: cache of known projects + * @param errorCache: keeps track of the modules whose project definition could not be found */ case class Resolution(rootDependencies: Set[Dependency], dependencies: Set[Dependency], conflicts: Set[Dependency], - projectsCache: Map[Resolution.ModuleVersion, (Repository, Project)], - errors: Map[Resolution.ModuleVersion, Seq[String]], + projectCache: Map[Resolution.ModuleVersion, (Repository, Project)], + errorCache: Map[Resolution.ModuleVersion, Seq[String]], filter: Option[Dependency => Boolean], profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) { import Resolution._ @@ -337,7 +337,7 @@ case class Resolution(rootDependencies: Set[Dependency], private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]() private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized { finalDependenciesCache.getOrElseUpdate(dep, - projectsCache.get(dep.moduleVersion) match { + projectCache.get(dep.moduleVersion) match { case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter) case None => Nil } @@ -374,7 +374,7 @@ case class Resolution(rootDependencies: Set[Dependency], val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion) (modules ++ nextModules) - .filterNot(mod => projectsCache.contains(mod) || errors.contains(mod)) + .filterNot(mod => projectCache.contains(mod) || errorCache.contains(mod)) } @@ -477,7 +477,7 @@ case class Resolution(rootDependencies: Set[Dependency], def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = { val approxProperties = project.parent - .flatMap(projectsCache.get) + .flatMap(projectCache.get) .map(_._2.properties) .fold(project.properties)(mergeProperties(project.properties, _)) @@ -511,13 +511,13 @@ case class Resolution(rootDependencies: Set[Dependency], if (toCheck.isEmpty) missing else if (toCheck.exists(done)) helper(toCheck -- done, done, missing) else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing) - else if (toCheck.exists(projectsCache.contains)) { - val (checking, remaining) = toCheck.partition(projectsCache.contains) - val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectsCache(mod)._2)) + else if (toCheck.exists(projectCache.contains)) { + val (checking, remaining) = toCheck.partition(projectCache.contains) + val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectCache(mod)._2)) helper(remaining ++ directRequirements, done ++ checking, missing) - } else if (toCheck.exists(errors.contains)) { - val (errored, remaining) = toCheck.partition(errors.contains) + } else if (toCheck.exists(errorCache.contains)) { + val (errored, remaining) = toCheck.partition(errorCache.contains) helper(remaining, done ++ errored, missing) } else helper(Set.empty, done, missing ++ toCheck) @@ -535,8 +535,8 @@ case class Resolution(rootDependencies: Set[Dependency], val approxProperties = project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) + .filter(projectCache.contains) + .map(projectCache(_)._2.properties) .fold(project.properties)(mergeProperties(project.properties, _)) val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) @@ -546,9 +546,9 @@ case class Resolution(rootDependencies: Set[Dependency], val deps = dependencies0 - .collect{ case dep if dep.scope == Scope.Import && projectsCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ - project.parent.filter(projectsCache.contains) - val projs = deps.map(projectsCache(_)._2) + .collect{ case dep if dep.scope == Scope.Import && projectCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ + project.parent.filter(projectCache.contains) + val projs = deps.map(projectCache(_)._2) val depMgmt = (project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement))) @@ -560,13 +560,13 @@ case class Resolution(rootDependencies: Set[Dependency], dependencies = dependencies0 .filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++ project.parent - .filter(projectsCache.contains) + .filter(projectCache.contains) .toSeq - .flatMap(projectsCache(_)._2.dependencies), + .flatMap(projectCache(_)._2.dependencies), dependencyManagement = depMgmt.values.toSeq, properties = project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) + .filter(projectCache.contains) + .map(projectCache(_)._2.properties) .fold(properties0)(mergeProperties(properties0, _)) ) } @@ -580,7 +580,7 @@ case class Resolution(rootDependencies: Set[Dependency], val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) gatheredLookups.flatMap{ lookupResults => - val errors0 = errors ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} + val errors0 = errorCache ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj} /* @@ -588,12 +588,12 @@ case class Resolution(rootDependencies: Set[Dependency], * dependency management / inheritance-related bits to them. */ - newProjects.foldLeft(Task.now(copy(errors = errors0))) { case (accTask, (modVer, (repo, proj))) => + newProjects.foldLeft(Task.now(copy(errorCache = errors0))) { case (accTask, (modVer, (repo, proj))) => for { current <- accTask updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule) proj0 = updated.withDependencyManagement(proj) - } yield updated.copy(projectsCache = updated.projectsCache + (modVer -> (repo, proj0))) + } yield updated.copy(projectCache = updated.projectCache + (modVer -> (repo, proj0))) } } } @@ -615,4 +615,17 @@ case class Resolution(rootDependencies: Set[Dependency], def minDependencies: Set[Dependency] = Orders.minDependencies(dependencies) + + def artifacts: Seq[Artifact] = + for { + dep <- minDependencies.toSeq + (repo, proj) <- projectCache.get(dep.moduleVersion).toSeq + artifact <- repo.artifacts(dep, proj) + } yield artifact + + def errors: Seq[(Dependency, Seq[String])] = + for { + dep <- dependencies.toSeq + err <- errorCache.get(dep.moduleVersion).toSeq + } yield (dep, err) } diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 87402dff0..2cfd39143 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -76,11 +76,11 @@ package object coursier { def apply(rootDependencies: Set[Dependency] = Set.empty, dependencies: Set[Dependency] = Set.empty, conflicts: Set[Dependency] = Set.empty, - projectsCache: Map[ModuleVersion, (Repository, Project)] = Map.empty, - errors: Map[ModuleVersion, Seq[String]] = Map.empty, + projectCache: Map[ModuleVersion, (Repository, Project)] = Map.empty, + errorCache: Map[ModuleVersion, Seq[String]] = Map.empty, filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution = - core.Resolution(rootDependencies, dependencies, conflicts, projectsCache, errors, filter, profileActivation) + core.Resolution(rootDependencies, dependencies, conflicts, projectCache, errorCache, filter, profileActivation) } def resolve(dependencies: Set[Dependency], @@ -98,4 +98,16 @@ package object coursier { startResolution.last(fetch, maxIterations.getOrElse(-1)) } + + type Artifact = core.Artifact + object Artifact { + def apply(url: String, + extra: Map[String, String] = Map.empty, + attributes: Attributes = Attributes()): Artifact = + core.Artifact(url, extra, attributes) + } + + type MavenRepository[G <: core.FetchMetadata] = core.MavenRepository[G] + val MavenRepository: core.MavenRepository.type = core.MavenRepository + } diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 3b36d9694..f8348043f 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -35,7 +35,7 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("ch.qos.logback", "logback-classic"), "1.1.3") val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) - .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here + .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( rootDependencies = Set(dep), @@ -51,7 +51,7 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("org.ow2.asm", "asm-commons"), "5.0.2") val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) - .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here + .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( rootDependencies = Set(dep), @@ -67,7 +67,7 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("joda-time", "joda-time"), "[2.2,2.8]") val res0 = await(resolve(Set(dep), fetchFrom(repositories)).runF) - val res = res0.copy(projectsCache = Map.empty, errors = Map.empty) + val res = res0.copy(projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -75,9 +75,9 @@ object CentralTests extends TestSuite { dep.withCompileScope)) assert(res == expected) - assert(res0.projectsCache.contains(dep.moduleVersion)) + assert(res0.projectCache.contains(dep.moduleVersion)) - val (_, proj) = res0.projectsCache(dep.moduleVersion) + val (_, proj) = res0.projectCache(dep.moduleVersion) assert(proj.version == "2.8") } } diff --git a/core/src/test/scala/coursier/test/ResolutionTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala index d10db84e5..4734bd0f5 100644 --- a/core/src/test/scala/coursier/test/ResolutionTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -168,7 +168,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - errors = Map(dep.moduleVersion -> Seq("Not found")) + errorCache = Map(dep.moduleVersion -> Seq("Not found")) ) assert(res == expected) @@ -185,7 +185,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - projectsCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) + projectCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) ) assert(res == expected) @@ -203,7 +203,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope, trDep.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv, projectsMap(trDep.moduleVersion).kv ) @@ -227,7 +227,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -252,7 +252,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -277,7 +277,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -297,7 +297,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ) @@ -316,7 +316,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -336,7 +336,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -355,7 +355,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -372,7 +372,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -391,7 +391,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -412,7 +412,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -435,7 +435,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -457,7 +457,7 @@ object ResolutionTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -481,7 +481,7 @@ object ResolutionTests extends TestSuite { deps, fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = deps, diff --git a/core/src/test/scala/coursier/test/TestRepository.scala b/core/src/test/scala/coursier/test/TestRepository.scala index b70077b34..fcdf892ff 100644 --- a/core/src/test/scala/coursier/test/TestRepository.scala +++ b/core/src/test/scala/coursier/test/TestRepository.scala @@ -12,8 +12,5 @@ class TestRepository(projects: Map[(Module, String), Project]) extends Repositor EitherT(Task.now( projects.get((module, version)).toRightDisjunction("Not found") )) - def versions(organization: String, name: String, cachePolicy: CachePolicy) = - EitherT(Task.now[String \/ Versions]( - -\/("Not supported") - )) + def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = ??? } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala new file mode 100644 index 000000000..014c56ea9 --- /dev/null +++ b/files/src/main/scala/coursier/Files.scala @@ -0,0 +1,121 @@ +package coursier + +import java.net.URL + +import coursier.core.CachePolicy + +import scala.annotation.tailrec +import scalaz.{-\/, \/-, \/, EitherT} +import scalaz.concurrent.Task + +import java.io._ + +// FIXME This kind of side-effecting API is lame, we should aim at a more functional one. +trait FilesLogger { + def foundLocally(f: File): Unit + def downloadingArtifact(url: String): Unit + def downloadedArtifact(url: String, success: Boolean): Unit +} + +case class Files(cache: Seq[(String, File)], + tmp: () => File, + logger: Option[FilesLogger] = None) { + + def file(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, File] = { + + cache.find{case (base, _) => artifact.url.startsWith(base)} match { + case None => ??? + case Some((base, cacheDir)) => + val file = new File(cacheDir, artifact.url.stripPrefix(base)) + + def locally = { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(file)) + \/-(file) + } + else -\/("Not found in cache") + } + } + + def remote = { + // FIXME A lot of things can go wrong here and are not properly handled: + // - checksums should be validated + // - what if the connection gets closed during the transfer (partial file on disk)? + // - what if someone is trying to write this file at the same time? (no locking of any kind yet) + // - ... + + Task { + try { + file.getParentFile.mkdirs() + + logger.foreach(_.downloadingArtifact(artifact.url)) + + val url = new URL(artifact.url) + val b = Array.fill[Byte](Files.bufferSize)(0) + val in = new BufferedInputStream(url.openStream(), Files.bufferSize) + + try { + val out = new FileOutputStream(file) + try { + @tailrec + def helper(): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + helper() + } + } + + helper() + } finally out.close() + } finally in.close() + + logger.foreach(_.downloadedArtifact(artifact.url, success = true)) + \/-(file) + } + catch { case e: Exception => + logger.foreach(_.downloadedArtifact(artifact.url, success = false)) + -\/(e.getMessage) + } + } + } + + EitherT(cachePolicy(locally)(remote)) + } + } + +} + +object Files { + + var bufferSize = 1024*1024 + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap(_.getMessage) + } + +} diff --git a/project/Coursier.scala b/project/Coursier.scala index 60e22572e..d11e0f0c7 100644 --- a/project/Coursier.scala +++ b/project/Coursier.scala @@ -101,8 +101,20 @@ object CoursierBuild extends Build { ) .enablePlugins(ScalaJSPlugin) - lazy val cli = Project(id = "cli", base = file("cli")) + lazy val files = Project(id = "files", base = file("files")) .dependsOn(coreJvm) + .settings(commonSettings: _*) + .settings( + name := "coursier-files", + libraryDependencies ++= Seq( + "org.http4s" %% "http4s-blazeclient" % "0.8.2", + "com.lihaoyi" %% "utest" % "0.3.0" % "test" + ), + testFrameworks += new TestFramework("utest.runner.Framework") + ) + + lazy val cli = Project(id = "cli", base = file("cli")) + .dependsOn(coreJvm, files) .settings(commonSettings ++ packAutoSettings ++ publishPackTxzArchive ++ publishPackZipArchive: _*) .settings( packArchivePrefix := s"coursier-cli_${scalaBinaryVersion.value}", diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 3cad52b95..528e4cd04 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,7 +1,7 @@ package coursier package web -import coursier.core.{Logger, Remote} +import coursier.core.{DefaultFetchMetadata, Logger} import japgolly.scalajs.react.vdom.{TagMod, Attr} import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope} @@ -18,7 +18,7 @@ case class ResolutionOptions(followOptional: Boolean = false, keepTest: Boolean = false) case class State(modules: Seq[Dependency], - repositories: Seq[Remote], + repositories: Seq[MavenRepository[DefaultFetchMetadata]], options: ResolutionOptions, resolutionOpt: Option[Resolution], editModuleIdx: Int, @@ -71,7 +71,7 @@ class Backend($: BackendScope[Unit, State]) { def updateTree(resolution: Resolution, target: String, reverse: Boolean) = { def depsOf(dep: Dependency) = - resolution.projectsCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter)) + resolution.projectCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter)) val minDependencies = resolution.minDependencies @@ -127,7 +127,7 @@ class Backend($: BackendScope[Unit, State]) { filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) ) - res.last(fetchFrom(s.repositories.map(_.copy(logger = Some(logger)))), 100) + res.last(fetchFrom(s.repositories.map(r => r.copy(fetchMetadata = r.fetchMetadata.copy(logger = Some(logger))))), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context @@ -224,7 +224,7 @@ object App { def depItem(dep: Dependency, finalVersionOpt: Option[String]) = { <.tr( - ^.`class` := (if (res.errors.contains(dep.moduleVersion)) "danger" else ""), + ^.`class` := (if (res.errorCache.contains(dep.moduleVersion)) "danger" else ""), <.td(dep.module.organization), <.td(dep.module.name), <.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")), @@ -234,11 +234,11 @@ object App { if (dep.attributes.classifier.isEmpty) Seq() else Seq(infoLabel(dep.attributes.classifier)), Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq, if (dep.optional) Seq(infoLabel("optional")) else Seq(), - res.errors.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq + res.errorCache.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq )), <.td(Seq[Seq[TagMod]]( - res.projectsCache.get(dep.moduleVersion) match { - case Some((repo: Remote, _)) => + res.projectCache.get(dep.moduleVersion) match { + case Some((MavenRepository(fetchMetadata, _), _)) => // FIXME Maven specific, generalize if/when adding support for Ivy val version0 = finalVersionOpt getOrElse dep.version val relPath = @@ -249,10 +249,10 @@ object App { ) Seq( - <.a(^.href := s"${repo.base}${relPath.mkString("/")}.pom", + <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.pom", <.span(^.`class` := "label label-info", "POM") ), - <.a(^.href := s"${repo.base}${relPath.mkString("/")}.jar", + <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.jar", <.span(^.`class` := "label label-info", "JAR") ) ) @@ -277,7 +277,7 @@ object App { ) ), <.tbody( - sortedDeps.map(dep => depItem(dep, res.projectsCache.get(dep.moduleVersion).map(_._2.version).filter(_ != dep.version))) + sortedDeps.map(dep => depItem(dep, res.projectCache.get(dep.moduleVersion).map(_._2.version).filter(_ != dep.version))) ) ) } @@ -386,19 +386,19 @@ object App { val modules = dependenciesTable("Dependencies") - val repositories = ReactComponentB[Seq[Remote]]("Repositories") + val repositories = ReactComponentB[Seq[MavenRepository[DefaultFetchMetadata]]]("Repositories") .render{ repos => - def repoItem(repo: Remote) = + def repoItem(repo: MavenRepository[DefaultFetchMetadata]) = <.tr( <.td( - <.a(^.href := repo.base, - repo.base + <.a(^.href := repo.fetchMetadata.root, + repo.fetchMetadata.root ) ) ) val sortedRepos = repos - .sortBy(repo => repo.base) + .sortBy(repo => repo.fetchMetadata.root) <.table(^.`class` := "table", <.thead(