diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 2d2c292f7..3f06b9891 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -5,28 +5,28 @@ import java.io.File import caseapp._ import coursier.core.{CachePolicy, Parse} -import coursier.core.{ArtifactDownloaderLogger, RemoteLogger, ArtifactDownloader} +import coursier.core.MetadataFetchLogger import scalaz.concurrent.Task -import scalaz.{-\/, \/-} case class Coursier(scope: List[String], keepOptional: Boolean, fetch: Boolean, - @ExtraName("N") maxIterations: Int) extends App { + @ExtraName("N") maxIterations: Int = 100) extends App { val scopes0 = if (scope.isEmpty) List(Scope.Compile, Scope.Runtime) else scope.map(Parse.scope) val scopes = scopes0.toSet - val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/central") + val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/metadata/central") + val centralFilesCacheDir = new File(sys.props("user.home") + "/.coursier/cache/files/central") val base = centralCacheDir.toURI def fileRepr(f: File) = base.relativize(f.toURI).getPath - val logger: RemoteLogger with ArtifactDownloaderLogger = new RemoteLogger with ArtifactDownloaderLogger { + val logger: MetadataFetchLogger with FilesLogger = new MetadataFetchLogger with FilesLogger { def println(s: String) = Console.err.println(s) def downloading(url: String) = @@ -53,44 +53,45 @@ case class Coursier(scope: List[String], ) } - val cachedMavenCentral = repository.mavenCentral.copy(cache = Some(centralCacheDir), logger = Some(logger)) + val cachedMavenCentral = repository.mavenCentral.copy( + fetchMetadata = repository.mavenCentral.fetchMetadata.copy( + cache = Some(centralCacheDir), + logger = Some(logger) + ) + ) val repositories = Seq[Repository]( cachedMavenCentral ) - lazy val downloaders = Map[Repository, ArtifactDownloader]( - cachedMavenCentral -> ArtifactDownloader(repository.mavenCentral.root, centralCacheDir, logger = Some(logger)) - ) - - val (splitArtifacts, malformed) = remainingArgs.toList + val (splitDependencies, malformed) = remainingArgs.toList .map(_.split(":", 3).toSeq) .partition(_.length == 3) - if (splitArtifacts.isEmpty) { - Console.err.println("Usage: coursier artifacts...") + if (splitDependencies.isEmpty) { + Console.err.println("Usage: coursier dependencies...") sys exit 1 } if (malformed.nonEmpty) { - Console.err.println(s"Malformed artifacts:\n${malformed.map(_.mkString(":")).mkString("\n")}") + Console.err.println(s"Malformed dependencies:\n${malformed.map(_.mkString(":")).mkString("\n")}") sys exit 1 } - val modules = splitArtifacts.map{ + val moduleVersions = splitDependencies.map{ case Seq(org, name, version) => (Module(org, name), version) } - val deps = modules.map{case (mod, ver) => + val deps = moduleVersions.map{case (mod, ver) => Dependency(mod, ver, scope = Scope.Runtime) } - val res = resolve( + val startRes = Resolution( deps.toSet, - fetchFrom(repositories), - maxIterations = Some(maxIterations).filter(_ > 0), filter = Some(dep => (keepOptional || !dep.optional) && scopes(dep.scope)) - ).run + ) + + val res = startRes.last(fetchFrom(repositories), maxIterations).run if (!res.isDone) { Console.err.println(s"Maximum number of iteration reached!") @@ -98,20 +99,16 @@ case class Coursier(scope: List[String], } def repr(dep: Dependency) = { - val (type0, classifier) = dep.artifacts match { - case maven: Artifacts.Maven => (maven.`type`, maven.classifier) - } - // dep.version can be an interval, whereas the one from project can't - val version = res.projectsCache.get(dep.moduleVersion).map(_._2.version).getOrElse(dep.version) + val version = res.projectCache.get(dep.moduleVersion).map(_._2.version).getOrElse(dep.version) val extra = if (version == dep.version) "" else s" ($version for ${dep.version})" - s"${dep.module.organization}:${dep.module.name}:$type0:${Some(classifier).filter(_.nonEmpty).map(_+":").mkString}$version$extra" + s"${dep.module.organization}:${dep.module.name}:${dep.attributes.`type`}:${Some(dep.attributes.classifier).filter(_.nonEmpty).map(_+":").mkString}$version$extra" } - val trDeps = res.dependencies.toList.sortBy(repr) + val trDeps = res.minDependencies.toList.sortBy(repr) println("\n" + trDeps.map(repr).distinct.mkString("\n")) @@ -120,11 +117,11 @@ case class Coursier(scope: List[String], println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") } - val errDeps = trDeps.filter(dep => res.errors.contains(dep.moduleVersion)) - if (errDeps.nonEmpty) { - println(s"${errDeps.size} error(s):") - for (dep <- errDeps) { - println(s" ${dep.module}:\n ${res.errors(dep.moduleVersion).mkString("\n").replace("\n", " \n")}") + val errors = res.errors + if (errors.nonEmpty) { + println(s"${errors.size} error(s):") + for ((dep, errs) <- errors) { + println(s" ${dep.module}:\n ${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") } } @@ -133,38 +130,11 @@ case class Coursier(scope: List[String], val cachePolicy: CachePolicy = CachePolicy.Default - val m = res.dependencies.groupBy(dep => res.projectsCache.get(dep.moduleVersion).map(_._1)) - val (notFound, remaining0) = m.partition(_._1.isEmpty) - if (notFound.nonEmpty) { - val notFound0 = notFound.values.flatten.toList.map(repr).sorted - println(s"Not found:${notFound0.mkString("\n")}") - } + val artifacts = res.artifacts - val (remaining, downloaderNotFound) = remaining0.partition(t => downloaders.contains(t._1.get)) - if (downloaderNotFound.nonEmpty) { - val downloaderNotFound0 = downloaderNotFound.values.flatten.toList.map(repr).sorted - println(s"Don't know how to download:${downloaderNotFound0.mkString("\n")}") - } - - val sorted = remaining - .toList - .map{ case (Some(repo), deps) => repo -> deps.toList.sortBy(repr) } - .sortBy(_._1.toString) // ... - - val tasks = - for { - (repo, deps) <- sorted - dl = downloaders(repo) - dep <- deps - (_, proj) = res.projectsCache(dep.moduleVersion) - } yield { - dl.artifacts(dep, proj, cachePolicy = cachePolicy).map { results => - val errorCount = results.count{case -\/(_) => true; case _ => false} - val resultsRepr = results.map(_.map(fileRepr).merge).map(" " + _).mkString("\n") - println(s"${repr(dep)} (${results.length} artifact(s)${if (errorCount > 0) s", $errorCount error(s)" else ""}):\n$resultsRepr") - } - } + val files = new Files(Seq(cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir), () => ???, Some(logger)) + val tasks = artifacts.map(files.file(_, cachePolicy).run) val task = Task.gatherUnordered(tasks) task.run diff --git a/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala new file mode 100644 index 000000000..d41b995bf --- /dev/null +++ b/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala @@ -0,0 +1,94 @@ +package coursier +package core + +import org.scalajs.dom.raw.{Event, XMLHttpRequest} + +import scala.concurrent.{ExecutionContext, Promise, Future} +import scalaz.{-\/, \/-, EitherT} +import scalaz.concurrent.Task + +import scala.scalajs.js +import js.Dynamic.{global => g} + +import scala.scalajs.js.timers._ + +object DefaultFetchMetadata { + + def encodeURIComponent(s: String): String = + g.encodeURIComponent(s).asInstanceOf[String] + + lazy val jsonpAvailable = !js.isUndefined(g.jsonp) + + /** Available if we're running on node, and package xhr2 is installed */ + lazy val xhr = g.require("xhr2") + def xhrReq() = + js.Dynamic.newInstance(xhr)().asInstanceOf[XMLHttpRequest] + + def fetchTimeout(target: String, p: Promise[_]) = + setTimeout(5000) { + if (!p.isCompleted) { + p.failure(new Exception(s"Timeout when fetching $target")) + } + } + + // FIXME Take into account HTTP error codes from YQL response + def proxiedJsonp(url: String)(implicit executionContext: ExecutionContext): Future[String] = { + val url0 = + "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20xml%20where%20url%3D%22" + + encodeURIComponent(url) + + "%22&format=jsonp&diagnostics=true" + + val p = Promise[String]() + + g.jsonp(url0, (res: js.Dynamic) => if (!p.isCompleted) { + val success = !js.isUndefined(res) && !js.isUndefined(res.results) + if (success) + p.success(res.results.asInstanceOf[js.Array[String]].mkString("\n")) + else + p.failure(new Exception(s"Fetching $url ($url0)")) + }) + + fetchTimeout(s"$url ($url0)", p) + p.future + } + + def get(url: String)(implicit executionContext: ExecutionContext): Future[String] = + if (jsonpAvailable) + proxiedJsonp(url) + else { + val p = Promise[String]() + val xhrReq0 = xhrReq() + val f = { _: Event => + p.success(xhrReq0.responseText) + } + xhrReq0.onload = f + + xhrReq0.open("GET", url) + xhrReq0.send() + + fetchTimeout(url, p) + p.future + } + +} + +trait Logger { + def fetching(url: String): Unit + def fetched(url: String): Unit + def other(url: String, msg: String): Unit +} + +case class DefaultFetchMetadata(root: String, + logger: Option[Logger] = None) extends FetchMetadata { + def apply(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, String] = { + + EitherT( + Task { implicit ec => + DefaultFetchMetadata.get(root + artifact.url) + .map(\/-(_)) + .recover{case e: Exception => -\/(e.getMessage)} + } + ) + } +} \ No newline at end of file diff --git a/core-js/src/main/scala/coursier/core/Remote.scala b/core-js/src/main/scala/coursier/core/Remote.scala deleted file mode 100644 index 6aa2d2700..000000000 --- a/core-js/src/main/scala/coursier/core/Remote.scala +++ /dev/null @@ -1,138 +0,0 @@ -package coursier -package core - -import org.scalajs.dom.raw.{Event, XMLHttpRequest} - -import scala.concurrent.{ExecutionContext, Promise, Future} -import scalaz.{-\/, \/-, \/, EitherT} -import scalaz.concurrent.Task - -import scala.scalajs.js -import js.Dynamic.{global => g} - -import scala.scalajs.js.timers._ - -object Remote { - - def encodeURIComponent(s: String): String = - g.encodeURIComponent(s).asInstanceOf[String] - - lazy val jsonpAvailable = !js.isUndefined(g.jsonp) - - /** Available if we're running on node, and package xhr2 is installed */ - lazy val xhr = g.require("xhr2") - def xhrReq() = - js.Dynamic.newInstance(xhr)().asInstanceOf[XMLHttpRequest] - - def fetchTimeout(target: String, p: Promise[_]) = - setTimeout(5000) { - if (!p.isCompleted) { - p.failure(new Exception(s"Timeout when fetching $target")) - } - } - - def proxiedJsonp(url: String)(implicit executionContext: ExecutionContext): Future[String] = { - val url0 = - "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20xml%20where%20url%3D%22" + - encodeURIComponent(url) + - "%22&format=jsonp&diagnostics=true" - - val p = Promise[String]() - - g.jsonp(url0, (res: js.Dynamic) => if (!p.isCompleted) { - val success = !js.isUndefined(res) && !js.isUndefined(res.results) - if (success) - p.success(res.results.asInstanceOf[js.Array[String]].mkString("\n")) - else - p.failure(new Exception(s"Fetching $url ($url0)")) - }) - - fetchTimeout(s"$url ($url0)", p) - p.future - } - - def get(url: String)(implicit executionContext: ExecutionContext): Future[Either[String, Xml.Node]] = - if (jsonpAvailable) - proxiedJsonp(url).map(compatibility.xmlParse) - else { - val p = Promise[Either[String, Xml.Node]]() - val xhrReq0 = xhrReq() - val f = { _: Event => - p.success(compatibility.xmlParse(xhrReq0.responseText)) - } - xhrReq0.onload = f - - xhrReq0.open("GET", url) - xhrReq0.send() - - fetchTimeout(url, p) - p.future - } - -} - -trait Logger { - def fetching(url: String): Unit - def fetched(url: String): Unit - def other(url: String, msg: String): Unit -} - -case class Remote(base: String, logger: Option[Logger] = None) extends MavenRepository { - - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { - - val path = { - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version.pom" - ) - } .map(Remote.encodeURIComponent) - - val url = base + path.mkString("/") - - EitherT(Task{ implicit ec => - logger.foreach(_.fetching(url)) - Remote.get(url).recover{case e: Exception => Left(e.getMessage)}.map{ eitherXml => - logger.foreach(_.fetched(url)) - for { - xml <- \/.fromEither(eitherXml) - _ = logger.foreach(_.other(url, "is XML")) - _ <- if (xml.label == "project") \/-(()) else -\/(s"Project definition not found (got '${xml.label}')") - _ = logger.foreach(_.other(url, "project definition found")) - proj <- Xml.project(xml) - _ = logger.foreach(_.other(url, "project definition ok")) - } yield proj - } - }) - } - - def versions(organization: String, - name: String, - cachePolicy: CachePolicy): EitherT[Task, String, Versions] = { - - val path = { - organization.split('.').toSeq ++ Seq( - name, - "maven-metadata.xml" - ) - } .map(Remote.encodeURIComponent) - - val url = base + path.mkString("/") - - EitherT(Task{ implicit ec => - logger.foreach(_.fetching(url)) - Remote.get(url).recover{case e: Exception => Left(e.getMessage)}.map{ eitherXml => - logger.foreach(_.fetched(url)) - for { - xml <- \/.fromEither(eitherXml) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - versions <- Xml.versions(xml) - } yield versions - } - }) - } - -} diff --git a/core-js/src/main/scala/coursier/core/compatibility/package.scala b/core-js/src/main/scala/coursier/core/compatibility/package.scala index 4c57573d7..5c940cb9d 100644 --- a/core-js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core-js/src/main/scala/coursier/core/compatibility/package.scala @@ -1,6 +1,7 @@ package coursier.core import scala.scalajs.js +import js.Dynamic.{global => g} import org.scalajs.dom.raw.NodeList package object compatibility { @@ -94,4 +95,7 @@ package object compatibility { Right(doc.fold(Xml.Node.empty)(fromNode)) } + def encodeURIComponent(s: String): String = + g.encodeURIComponent(s).asInstanceOf[String] + } diff --git a/core-js/src/main/scala/coursier/repository/package.scala b/core-js/src/main/scala/coursier/repository/package.scala deleted file mode 100644 index 59d758a39..000000000 --- a/core-js/src/main/scala/coursier/repository/package.scala +++ /dev/null @@ -1,13 +0,0 @@ -package coursier - -package object repository { - - type Remote = core.Remote - val Remote: core.Remote.type = core.Remote - - val mavenCentral = Remote("https://repo1.maven.org/maven2/") - - val sonatypeReleases = Remote("https://oss.sonatype.org/content/repositories/releases/") - val sonatypeSnapshots = Remote("https://oss.sonatype.org/content/repositories/snapshots/") - -} diff --git a/core-js/src/test/scala/coursier/test/JsTests.scala b/core-js/src/test/scala/coursier/test/JsTests.scala index 5888d9331..9738b2600 100644 --- a/core-js/src/test/scala/coursier/test/JsTests.scala +++ b/core-js/src/test/scala/coursier/test/JsTests.scala @@ -1,7 +1,7 @@ package coursier package test -import coursier.core.Remote +import coursier.core.DefaultFetchMetadata import coursier.test.compatibility._ import utest._ @@ -18,7 +18,8 @@ object JsTests extends TestSuite { } 'get{ - Remote.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + DefaultFetchMetadata.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + .map(core.compatibility.xmlParse) .map{ xml => assert(xml.right.toOption.exists(_.label == "project")) } diff --git a/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala new file mode 100644 index 000000000..30f7f87af --- /dev/null +++ b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala @@ -0,0 +1,101 @@ +package coursier +package core + +import java.io._ +import java.net.URL + +import scala.io.Codec +import scalaz._, Scalaz._ +import scalaz.concurrent.Task + +trait MetadataFetchLogger { + def downloading(url: String): Unit + def downloaded(url: String, success: Boolean): Unit + def readingFromCache(f: File): Unit + def puttingInCache(f: File): Unit +} + +case class DefaultFetchMetadata(root: String, + cache: Option[File] = None, + logger: Option[MetadataFetchLogger] = None) extends FetchMetadata { + + def apply(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, String] = { + lazy val localFile = { + for { + cache0 <- cache.toRightDisjunction("No cache") + f = new File(cache0, artifact.url) + } yield f + } + + def locally = { + Task { + for { + f0 <- localFile + f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") + content <- \/.fromTryCatchNonFatal{ + logger.foreach(_.readingFromCache(f)) + scala.io.Source.fromFile(f)(Codec.UTF8).mkString + }.leftMap(_.getMessage) + } yield content + } + } + + def remote = { + val urlStr = root + artifact.url + val url = new URL(urlStr) + + def log = Task(logger.foreach(_.downloading(urlStr))) + def get = DefaultFetchMetadata.readFully(url.openStream()) + + log.flatMap(_ => get) + } + + def save(s: String) = { + localFile.fold(_ => Task.now(()), f => + Task { + if (!f.exists()) { + logger.foreach(_.puttingInCache(f)) + f.getParentFile.mkdirs() + val w = new PrintWriter(f) + try w.write(s) + finally w.close() + () + } + } + ) + } + + EitherT(cachePolicy.saving(locally)(remote)(save)) + } + +} + +object DefaultFetchMetadata { + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap(_.getMessage) + } + +} diff --git a/core-jvm/src/main/scala/coursier/core/Remote.scala b/core-jvm/src/main/scala/coursier/core/Remote.scala deleted file mode 100644 index f9008f2b4..000000000 --- a/core-jvm/src/main/scala/coursier/core/Remote.scala +++ /dev/null @@ -1,254 +0,0 @@ -package coursier -package core - -import java.io._ -import java.net.URL - -import scala.annotation.tailrec -import scala.io.Codec -import scalaz._, Scalaz._ -import scalaz.concurrent.Task - -// FIXME This kind of side-effecting API is lame, we should aim at a more functional one. -trait ArtifactDownloaderLogger { - def foundLocally(f: File): Unit - def downloadingArtifact(url: String): Unit - def downloadedArtifact(url: String, success: Boolean): Unit -} - -case class ArtifactDownloader(root: String, cache: File, logger: Option[ArtifactDownloaderLogger] = None) { - var bufferSize = 1024*1024 - - def artifact(module: Module, - version: String, - artifact: Artifacts.Artifact, - cachePolicy: CachePolicy): EitherT[Task, String, File] = { - - val relPath = - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version${Some(artifact.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${artifact.`type`}" - ) - - val file = (cache /: relPath)(new File(_, _)) - - def locally = { - Task { - if (file.exists()) { - logger.foreach(_.foundLocally(file)) - \/-(file) - } - else -\/("Not found in cache") - } - } - - def remote = { - // FIXME A lot of things can go wrong here and are not properly handled: - // - checksums should be validated - // - what if the connection gets closed during the transfer (partial file on disk)? - // - what if someone is trying to write this file at the same time? (no locking of any kind yet) - // - ... - - val urlStr = root + relPath.mkString("/") - - Task { - try { - file.getParentFile.mkdirs() - - logger.foreach(_.downloadingArtifact(urlStr)) - - val url = new URL(urlStr) - val b = Array.fill[Byte](bufferSize)(0) - val in = new BufferedInputStream(url.openStream(), bufferSize) - - try { - val out = new FileOutputStream(file) - try { - @tailrec - def helper(): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - helper() - } - } - - helper() - } finally out.close() - } finally in.close() - - logger.foreach(_.downloadedArtifact(urlStr, success = true)) - \/-(file) - } - catch { case e: Exception => - logger.foreach(_.downloadedArtifact(urlStr, success = false)) - -\/(e.getMessage) - } - } - } - - EitherT(cachePolicy(locally)(remote)) - } - - def artifacts(dependency: Dependency, - project: Project, - cachePolicy: CachePolicy = CachePolicy.Default): Task[Seq[String \/ File]] = { - - val artifacts0 = - dependency.artifacts match { - case s: Artifacts.Sufficient => s.artifacts - case p: Artifacts.WithProject => p.artifacts(project) - } - - val tasks = - artifacts0 .map { artifact0 => - // Important: using version from project, as the one from dependency can be an interval - artifact(dependency.module, project.version, artifact0, cachePolicy = cachePolicy).run - } - - Task.gatherUnordered(tasks) - } - -} - -// FIXME Comment of ArtifactDownloaderLogger applies here too -trait RemoteLogger { - def downloading(url: String): Unit - def downloaded(url: String, success: Boolean): Unit - def readingFromCache(f: File): Unit - def puttingInCache(f: File): Unit -} - -object Remote { - - def readFullySync(is: InputStream) = { - val buffer = new ByteArrayOutputStream() - val data = Array.ofDim[Byte](16384) - - var nRead = is.read(data, 0, data.length) - while (nRead != -1) { - buffer.write(data, 0, nRead) - nRead = is.read(data, 0, data.length) - } - - buffer.flush() - buffer.toByteArray - } - - def readFully(is: => InputStream) = - Task { - \/.fromTryCatchNonFatal { - val is0 = is - val b = - try readFullySync(is) - finally is0.close() - - new String(b, "UTF-8") - } .leftMap(_.getMessage) - } - -} - -case class Remote(root: String, - cache: Option[File] = None, - logger: Option[RemoteLogger] = None) extends MavenRepository { - - private def get(path: Seq[String], - cachePolicy: CachePolicy): EitherT[Task, String, String] = { - - lazy val localFile = { - for { - cache0 <- cache.toRightDisjunction("No cache") - f = (cache0 /: path)(new File(_, _)) - } yield f - } - - def locally = { - Task { - for { - f0 <- localFile - f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") - content <- \/.fromTryCatchNonFatal{ - logger.foreach(_.readingFromCache(f)) - scala.io.Source.fromFile(f)(Codec.UTF8).mkString - }.leftMap(_.getMessage) - } yield content - } - } - - def remote = { - val urlStr = root + path.mkString("/") - val url = new URL(urlStr) - - def log = Task(logger.foreach(_.downloading(urlStr))) - def get = Remote.readFully(url.openStream()) - - log.flatMap(_ => get) - } - - def save(s: String) = { - localFile.fold(_ => Task.now(()), f => - Task { - if (!f.exists()) { - logger.foreach(_.puttingInCache(f)) - f.getParentFile.mkdirs() - val w = new PrintWriter(f) - try w.write(s) - finally w.close() - () - } - } - ) - } - - EitherT(cachePolicy.saving(locally)(remote)(save)) - } - - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { - - val path = - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-$version.pom" - ) - - val task = get(path, cachePolicy).run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") - proj <- Xml.project(xml) - } yield proj - ) - - EitherT(task) - } - - def versions(organization: String, - name: String, - cachePolicy: CachePolicy): EitherT[Task, String, Versions] = { - - val path = - organization.split('.').toSeq ++ Seq( - name, - "maven-metadata.xml" - ) - - val task = get(path, cachePolicy).run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - versions <- Xml.versions(xml) - } yield versions - ) - - EitherT(task) - } -} diff --git a/core-jvm/src/main/scala/coursier/core/compatibility/package.scala b/core-jvm/src/main/scala/coursier/core/compatibility/package.scala index 5eb8cbc97..eb4cb154c 100644 --- a/core-jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core-jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -25,4 +25,6 @@ package object compatibility { .map(fromNode) } + def encodeURIComponent(s: String): String = + new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString } diff --git a/core-jvm/src/main/scala/coursier/core/package.scala b/core-jvm/src/main/scala/coursier/core/package.scala index aaf241b9f..3d7851006 100644 --- a/core-jvm/src/main/scala/coursier/core/package.scala +++ b/core-jvm/src/main/scala/coursier/core/package.scala @@ -11,7 +11,7 @@ package object core { profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]): Stream[Resolution] = { val startResolution = Resolution( - dependencies, dependencies, Set.empty, + dependencies, Set.empty, Set.empty, Map.empty, Map.empty, filter, profileActivation diff --git a/core-jvm/src/main/scala/coursier/repository/package.scala b/core-jvm/src/main/scala/coursier/repository/package.scala deleted file mode 100644 index 59d758a39..000000000 --- a/core-jvm/src/main/scala/coursier/repository/package.scala +++ /dev/null @@ -1,13 +0,0 @@ -package coursier - -package object repository { - - type Remote = core.Remote - val Remote: core.Remote.type = core.Remote - - val mavenCentral = Remote("https://repo1.maven.org/maven2/") - - val sonatypeReleases = Remote("https://oss.sonatype.org/content/repositories/releases/") - val sonatypeSnapshots = Remote("https://oss.sonatype.org/content/repositories/snapshots/") - -} diff --git a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala index 250a46515..10c93544c 100644 --- a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala +++ b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.core.Remote +import coursier.core.DefaultFetchMetadata import scala.concurrent.{ExecutionContext, Future} import scalaz.concurrent.Task @@ -17,7 +17,7 @@ package object compatibility { def is = getClass.getClassLoader .getResource(path).openStream() - new String(Remote.readFullySync(is), "UTF-8") + new String(DefaultFetchMetadata.readFullySync(is), "UTF-8") } } diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index ee48bd04e..75c775633 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -32,36 +32,15 @@ sealed abstract class Scope(val name: String) case class Dependency(module: Module, version: String, scope: Scope, - artifacts: Artifacts, + attributes: Attributes, exclusions: Set[(String, String)], optional: Boolean) { def moduleVersion = (module, version) } -sealed trait Artifacts - -object Artifacts { - /** - * May become a bit more complicated with Ivy support, - * but should still point at one single artifact. - */ - case class Artifact(`type`: String, +case class Attributes(`type`: String, classifier: String) - sealed trait WithProject extends Artifacts { - def artifacts(project: Project): Seq[Artifact] - } - - sealed trait Sufficient extends Artifacts { - def artifacts: Seq[Artifact] - } - - case class Maven(`type`: String, - classifier: String) extends Sufficient { - def artifacts: Seq[Artifact] = Seq(Artifact(`type`, classifier)) - } -} - case class Project(module: Module, version: String, dependencies: Seq[Dependency], @@ -91,6 +70,7 @@ case class Profile(id: String, dependencyManagement: Seq[Dependency], properties: Map[String, String]) +// FIXME Move to MavenRepository? case class Versions(latest: String, release: String, available: List[String], @@ -99,3 +79,27 @@ case class Versions(latest: String, object Versions { case class DateTime(year: Int, month: Int, day: Int, hour: Int, minute: Int, second: Int) } + +case class Artifact(url: String, + extra: Map[String, String], + attributes: Attributes) + +object Artifact { + val md5 = "md5" + val sha1 = "sha1" + val sig = "pgp" + val sigMd5 = "md5-pgp" + val sigSha1 = "sha1-pgp" + val sources = "src" + val sourcesMd5 = "md5-src" + val sourcesSha1 = "sha1-src" + val sourcesSig = "src-pgp" + val sourcesSigMd5 = "md5-src-pgp" + val sourcesSigSha1 = "sha1-src-pgp" + val javadoc = "javadoc" + val javadocMd5 = "md5-javadoc" + val javadocSha1 = "sha1-javadoc" + val javadocSig = "javadoc-pgp" + val javadocSigMd5 = "md5-javadoc-pgp" + val javadocSigSha1 = "sha1-javadoc-pgp" +} diff --git a/core/src/main/scala/coursier/core/Exclusions.scala b/core/src/main/scala/coursier/core/Exclusions.scala new file mode 100644 index 000000000..997c9bc61 --- /dev/null +++ b/core/src/main/scala/coursier/core/Exclusions.scala @@ -0,0 +1,87 @@ +package coursier.core + +object Exclusions { + + def partition(exclusions: Set[(String, String)]): (Boolean, Set[String], Set[String], Set[(String, String)]) = { + + val (wildCards, remaining) = exclusions + .partition{case (org, name) => org == "*" || name == "*" } + + val all = wildCards + .contains(one.head) + + val excludeByOrg = wildCards + .collect{case (org, "*") if org != "*" => org } + val excludeByName = wildCards + .collect{case ("*", name) if name != "*" => name } + + (all, excludeByOrg, excludeByName, remaining) + } + + def apply(exclusions: Set[(String, String)]): (String, String) => Boolean = { + + val (all, excludeByOrg, excludeByName, remaining) = partition(exclusions) + + if (all) (_, _) => false + else + (org, name) => { + !excludeByName(name) && + !excludeByOrg(org) && + !remaining((org, name)) + } + } + + def minimize(exclusions: Set[(String, String)]): Set[(String, String)] = { + + val (all, excludeByOrg, excludeByName, remaining) = partition(exclusions) + + if (all) one + else { + val filteredRemaining = remaining + .filter{case (org, name) => + !excludeByOrg(org) && + !excludeByName(name) + } + + excludeByOrg.map((_, "*")) ++ + excludeByName.map(("*", _)) ++ + filteredRemaining + } + } + + val zero = Set.empty[(String, String)] + val one = Set(("*", "*")) + + def join(x: Set[(String, String)], y: Set[(String, String)]): Set[(String, String)] = + minimize(x ++ y) + + def meet(x: Set[(String, String)], y: Set[(String, String)]): Set[(String, String)] = { + + val ((xAll, xExcludeByOrg, xExcludeByName, xRemaining), (yAll, yExcludeByOrg, yExcludeByName, yRemaining)) = + (partition(x), partition(y)) + + val all = xAll && yAll + + if (all) one + else { + val excludeByOrg = + if (xAll) yExcludeByOrg + else if (yAll) xExcludeByOrg + else xExcludeByOrg intersect yExcludeByOrg + val excludeByName = + if (xAll) yExcludeByName + else if (yAll) xExcludeByName + else xExcludeByName intersect yExcludeByName + + val remaining = + xRemaining.filter{case (org, name) => yAll || yExcludeByOrg(org) || yExcludeByName(name)} ++ + yRemaining.filter{case (org, name) => xAll || xExcludeByOrg(org) || xExcludeByName(name)} ++ + (xRemaining intersect yRemaining) + + excludeByOrg.map((_, "*")) ++ + excludeByName.map(("*", _)) ++ + remaining + } + } + +} diff --git a/core/src/main/scala/coursier/core/Orders.scala b/core/src/main/scala/coursier/core/Orders.scala new file mode 100644 index 000000000..dd5c7a28a --- /dev/null +++ b/core/src/main/scala/coursier/core/Orders.scala @@ -0,0 +1,141 @@ +package coursier.core + +object Orders { + + /** Minimal ad-hoc partial order */ + trait PartialOrder[A] { + /** + * x < y: Some(neg. integer) + * x == y: Some(0) + * x > y: Some(pos. integer) + * x, y not related: None + */ + def cmp(x: A, y: A): Option[Int] + } + + /** + * Only relations: + * Compile < Runtime < Test + */ + implicit val mavenScopePartialOrder: PartialOrder[Scope] = + new PartialOrder[Scope] { + val higher = Map[Scope, Set[Scope]]( + Scope.Compile -> Set(Scope.Runtime, Scope.Test), + Scope.Runtime -> Set(Scope.Test) + ) + + def cmp(x: Scope, y: Scope) = + if (x == y) Some(0) + else if (higher.get(x).exists(_(y))) Some(-1) + else if (higher.get(y).exists(_(x))) Some(1) + else None + } + + /** Non-optional < optional */ + implicit val optionalPartialOrder: PartialOrder[Boolean] = + new PartialOrder[Boolean] { + def cmp(x: Boolean, y: Boolean) = + Some( + if (x == y) 0 + else if (x) 1 + else -1 + ) + } + + /** + * Exclusions partial order. + * + * x <= y iff all that x excludes is also excluded by y. + * x and y not related iff x excludes some elements not excluded by y AND + * y excludes some elements not excluded by x. + * + * In particular, no exclusions <= anything <= Set(("*", "*")) + */ + implicit val exclusionsPartialOrder: PartialOrder[Set[(String, String)]] = + new PartialOrder[Set[(String, String)]] { + def boolCmp(a: Boolean, b: Boolean) = (a, b) match { + case (true, true) => Some(0) + case (true, false) => Some(1) + case (false, true) => Some(-1) + case (false, false) => None + } + + def cmp(x: Set[(String, String)], y: Set[(String, String)]) = { + val (xAll, xExcludeByOrg1, xExcludeByName1, xRemaining0) = Exclusions.partition(x) + val (yAll, yExcludeByOrg1, yExcludeByName1, yRemaining0) = Exclusions.partition(y) + + boolCmp(xAll, yAll).orElse { + def filtered(e: Set[(String, String)]) = + e.filter{case (org, name) => + !xExcludeByOrg1(org) && !yExcludeByOrg1(org) && + !xExcludeByName1(name) && !yExcludeByName1(name) + } + + def removeIntersection[T](a: Set[T], b: Set[T]) = + (a -- b, b -- a) + + def allEmpty(set: Set[_]*) = set.forall(_.isEmpty) + + val (xRemaining1, yRemaining1) = + (filtered(xRemaining0), filtered(yRemaining0)) + + val (xProperRemaining, yProperRemaining) = + removeIntersection(xRemaining1, yRemaining1) + + val (onlyXExcludeByOrg, onlyYExcludeByOrg) = + removeIntersection(xExcludeByOrg1, yExcludeByOrg1) + + val (onlyXExcludeByName, onlyYExcludeByName) = + removeIntersection(xExcludeByName1, yExcludeByName1) + + val (noXProper, noYProper) = ( + allEmpty(xProperRemaining, onlyXExcludeByOrg, onlyXExcludeByName), + allEmpty(yProperRemaining, onlyYExcludeByOrg, onlyYExcludeByName) + ) + + boolCmp(noYProper, noXProper) // order matters + } + } + } + + /** + * Assume all dependencies have same `module`, `version`, and `artifact`; see `minDependencies` + * if they don't. + */ + def minDependenciesUnsafe(dependencies: Set[Dependency]): Set[Dependency] = { + val groupedDependencies = dependencies + .groupBy(dep => (dep.optional, dep.scope)) + .mapValues(deps => deps.head.copy(exclusions = deps.foldLeft(Exclusions.one)((acc, dep) => Exclusions.meet(acc, dep.exclusions)))) + .toList + + val remove = + for { + List(((xOpt, xScope), xDep), ((yOpt, yScope), yDep)) <- groupedDependencies.combinations(2) + optCmp <- optionalPartialOrder.cmp(xOpt, yOpt).iterator + scopeCmp <- mavenScopePartialOrder.cmp(xScope, yScope).iterator + if optCmp*scopeCmp >= 0 + exclCmp <- exclusionsPartialOrder.cmp(xDep.exclusions, yDep.exclusions).iterator + if optCmp*exclCmp >= 0 + if scopeCmp*exclCmp >= 0 + xIsMin = optCmp < 0 || scopeCmp < 0 || exclCmp < 0 + yIsMin = optCmp > 0 || scopeCmp > 0 || exclCmp > 0 + if xIsMin || yIsMin // should be always true, unless xDep == yDep, which shouldn't happen + } yield if (xIsMin) yDep else xDep + + groupedDependencies.map(_._2).toSet -- remove + } + + /** + * Minified representation of `dependencies`. + * + * The returned set brings exactly the same things as `dependencies`, with no redundancy. + */ + def minDependencies(dependencies: Set[Dependency]): Set[Dependency] = { + dependencies + .groupBy(_.copy(scope = Scope.Other(""), exclusions = Set.empty, optional = false)) + .mapValues(minDependenciesUnsafe) + .valuesIterator + .fold(Set.empty)(_ ++ _) + } + +} diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index c953d64cc..d95d32eef 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -3,9 +3,11 @@ package coursier.core import scalaz.{-\/, \/-, \/, EitherT} import scalaz.concurrent.Task +import coursier.core.compatibility.encodeURIComponent + trait Repository { def find(module: Module, version: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Project] - def versions(organization: String, name: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Versions] + def artifacts(dependency: Dependency, project: Project): Seq[Artifact] } sealed trait CachePolicy { @@ -37,7 +39,132 @@ object CachePolicy { } } -trait MavenRepository extends Repository { +object Repository { + implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { + def withDefaultChecksums: Artifact = + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.md5 -> (underlying.url + ".md5"), + Artifact.sha1 -> (underlying.url + ".sha1") + )) + def withDefaultSignature: Artifact = + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.sigMd5 -> (underlying.url + ".asc.md5"), + Artifact.sigSha1 -> (underlying.url + ".asc.sha1"), + Artifact.sig -> (underlying.url + ".asc") + )) + def withJavadocSources: Artifact = { + val base = underlying.url.stripSuffix(".jar") + underlying.copy(extra = underlying.extra ++ Seq( + Artifact.sourcesMd5 -> (base + "-sources.jar.md5"), + Artifact.sourcesSha1 -> (base + "-sources.jar.sha1"), + Artifact.sources -> (base + "-sources.jar"), + Artifact.sourcesSigMd5 -> (base + "-sources.jar.asc.md5"), + Artifact.sourcesSigSha1 -> (base + "-sources.jar.asc.sha1"), + Artifact.sourcesSig -> (base + "-sources.jar.asc"), + Artifact.javadocMd5 -> (base + "-javadoc.jar.md5"), + Artifact.javadocSha1 -> (base + "-javadoc.jar.sha1"), + Artifact.javadoc -> (base + "-javadoc.jar"), + Artifact.javadocSigMd5 -> (base + "-javadoc.jar.asc.md5"), + Artifact.javadocSigSha1 -> (base + "-javadoc.jar.asc.sha1"), + Artifact.javadocSig -> (base + "-javadoc.asc.jar") + )) + } + } +} + +trait FetchMetadata { + def root: String + def apply(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, String] +} + +case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, + ivyLike: Boolean = false) extends Repository { + + import Repository._ + + def projectArtifact(module: Module, version: String): Artifact = { + if (ivyLike) ??? + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + version, + s"${module.name}-$version.pom" + ) + ) .map(encodeURIComponent) + + Artifact( + path.mkString("/"), + Map( + Artifact.md5 -> "", + Artifact.sha1 -> "" + ), + Attributes("pom", "") + ) + .withDefaultSignature + } + } + + def versionsArtifact(module: Module): Option[Artifact] = + if (ivyLike) None + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + "maven-metadata.xml" + ) + ) .map(encodeURIComponent) + + val artifact = + Artifact( + path.mkString("/"), + Map.empty, + Attributes("pom", "") + ) + .withDefaultChecksums + + Some(artifact) + } + + def versions(module: Module, + cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Versions] = { + + EitherT( + versionsArtifact(module) match { + case None => Task.now(-\/("Not supported")) + case Some(artifact) => + fetchMetadata(artifact, cachePolicy) + .run + .map(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") + versions <- Xml.versions(xml) + } yield versions + ) + } + ) + } + + def findNoInterval(module: Module, + version: String, + cachePolicy: CachePolicy): EitherT[Task, String, Project] = { + + EitherT { + fetchMetadata(projectArtifact(module, version), cachePolicy) + .run + .map(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") + proj <- Xml.project(xml) + } yield proj + ) + } + } def find(module: Module, version: String, @@ -46,28 +173,55 @@ trait MavenRepository extends Repository { Parse.versionInterval(version).filter(_.isValid) match { case None => findNoInterval(module, version, cachePolicy) case Some(itv) => - versions(module.organization, module.name, cachePolicy).flatMap { versions0 => - val eitherVersion = { - val release = Version(versions0.release) - if (itv.contains(release)) \/-(versions0.release) - else { - val inInterval = versions0.available.map(Version(_)).filter(itv.contains) - if (inInterval.isEmpty) -\/(s"No version found for $version") - else \/-(inInterval.max.repr) + versions(module, cachePolicy) + .flatMap { versions0 => + val eitherVersion = { + val release = Version(versions0.release) + + if (itv.contains(release)) \/-(versions0.release) + else { + val inInterval = versions0.available + .map(Version(_)) + .filter(itv.contains) + + if (inInterval.isEmpty) -\/(s"No version found for $version") + else \/-(inInterval.max.repr) + } + } + + eitherVersion match { + case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason))) + case \/-(version0) => + findNoInterval(module, version0, cachePolicy) + .map(_.copy(versions = Some(versions0))) } } - - eitherVersion match { - case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason))) - case \/-(version0) => findNoInterval(module, version0, cachePolicy) - .map(_.copy(versions = Some(versions0))) - } - } } } - def findNoInterval(module: Module, - version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] + def artifacts(dependency: Dependency, + project: Project): Seq[Artifact] = { -} + val path = + dependency.module.organization.split('.').toSeq ++ Seq( + dependency.module.name, + project.version, + s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + ) + + var artifact = + Artifact( + fetchMetadata.root + path.mkString("/"), + Map.empty, + dependency.attributes + ) + .withDefaultChecksums + + if (dependency.attributes.`type` == "jar") + artifact = artifact + .withDefaultSignature + .withJavadocSources + + Seq(artifact) + } +} \ No newline at end of file diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala new file mode 100644 index 000000000..1f771a258 --- /dev/null +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -0,0 +1,631 @@ +package coursier.core + +import java.util.regex.Pattern.quote + +import scala.annotation.tailrec +import scala.collection.mutable +import scalaz.concurrent.Task +import scalaz.{EitherT, \/-, \/, -\/} + +object Resolution { + + type ModuleVersion = (Module, String) + + /** + * Try to find `module` among `repositories`. + * + * Look at `repositories` from the left, one-by-one, and stop at first success. + * Else, return all errors, in the same order. + * + * The `version` field of the returned `Project` in case of success may not be + * equal to the provided one, in case the latter is not a specific + * version (e.g. version interval). Which version get chosen depends on + * the repository implementation. + */ + def find(repositories: Seq[Repository], + module: Module, + version: String): EitherT[Task, List[String], (Repository, Project)] = { + + val lookups = repositories.map(repo => repo -> repo.find(module, version).run) + val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[List[String] \/ (Repository, Project)]) { + case (acc, (repo, t)) => + acc.flatMap { + case -\/(errors) => + t.map(res => res + .flatMap(project => + if (project.module == module) \/-((repo, project)) + else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") + ) + .leftMap(error => error :: errors) + ) + + case res @ \/-(_) => + Task.now(res) + } + } + + EitherT(task.map(_.leftMap(_.reverse))).map { case x @ (_, proj) => + assert(proj.module == module) + x + } + } + + /** + * Get the active profiles of `project`, using the current properties `properties`, + * and `profileActivation` stating if a profile is active. + */ + def profiles(project: Project, + properties: Map[String, String], + profileActivation: (String, Activation, Map[String, String]) => Boolean): Seq[Profile] = { + + val activated = project.profiles + .filter(p => profileActivation(p.id, p.activation, properties)) + + def default = project.profiles + .filter(_.activeByDefault.toSeq.contains(true)) + + if (activated.isEmpty) default + else activated + } + + type DepMgmtKey = (String, String, String) + def dependencyManagementKey(dep: Dependency): DepMgmtKey = + (dep.module.organization, dep.module.name, dep.attributes.`type`) + def dependencyManagementAdd(m: Map[DepMgmtKey, Dependency], dep: Dependency): Map[DepMgmtKey, Dependency] = { + val key = dependencyManagementKey(dep) + if (m.contains(key)) m else m + (key -> dep) + } + def dependencyManagementAddSeq(m: Map[DepMgmtKey, Dependency], deps: Seq[Dependency]): Map[DepMgmtKey, Dependency] = + (m /: deps)(dependencyManagementAdd) + + def mergeProperties(m: Map[String, String], other: Map[String, String]): Map[String, String] = { + m ++ other.filterKeys(!m.contains(_)) + } + + def addDependencies(deps: Seq[Seq[Dependency]]): Seq[Dependency] = { + val res = + deps.foldRight((Set.empty[DepMgmtKey], Seq.empty[Dependency])) { + case (deps0, (set, acc)) => + val deps = deps0.filter(dep => !set(dependencyManagementKey(dep))) + (set ++ deps.map(dependencyManagementKey), acc ++ deps) + } + + res._2 + } + + val propRegex = (quote("${") + "([a-zA-Z0-9-.]*)" + quote("}")).r + + /** + * Substitutes `properties` in `dependencies`. + */ + def withProperties(dependencies: Seq[Dependency], + properties: Map[String, String]): Seq[Dependency] = { + + def substituteProps(s: String) = { + val matches = propRegex.findAllMatchIn(s).toList.reverse + if (matches.isEmpty) s + else { + val output = (new StringBuilder(s) /: matches) { + (b, m) => properties.get(m.group(1)).fold(b)(b.replace(m.start, m.end, _)) + } + output.result() + } + } + + dependencies.map{ dep => + dep.copy( + module = dep.module.copy( + organization = substituteProps(dep.module.organization), + name = substituteProps(dep.module.name) + ), + version = substituteProps(dep.version), + attributes = dep.attributes.copy( + `type` = substituteProps(dep.attributes.`type`), + classifier = substituteProps(dep.attributes.classifier) + ), + scope = Parse.scope(substituteProps(dep.scope.name)), + exclusions = dep.exclusions + .map{case (org, name) => (substituteProps(org), substituteProps(name))} + // FIXME The content of the optional tag may also be a property in the original POM. + // Maybe not parse it that earlier? + ) + } + } + + /** + * Merge several version constraints together. Returns `None` in case of conflict. + */ + def mergeVersions(versions: Seq[String]): Option[String] = { + val (nonParsedConstraints, parsedConstraints) = + versions + .map(v => v -> Parse.versionConstraint(v)) + .partition(_._2.isEmpty) + + // FIXME Report this in return type, not this way + if (nonParsedConstraints.nonEmpty) + Console.err.println(s"Ignoring unparsed versions: ${nonParsedConstraints.map(_._1)}") + + val constraintOpt = + (Option(VersionInterval.zero) /: parsedConstraints.map(_._2.get.interval)) { + case (acc, itv) => acc.flatMap(_.merge(itv)) + } .map(_.constraint) + + constraintOpt.map(_.repr) + } + + /** + * Merge several dependencies, solving version constraints of duplicated modules. + * Returns the conflicted dependencies, and the (merged) others. + */ + def merge(dependencies: TraversableOnce[Dependency]): (Seq[Dependency], Seq[Dependency]) = { + val m = dependencies + .toList + .groupBy(dep => dep.module) + .mapValues{ deps => + if (deps.lengthCompare(1) == 0) \/-(deps) + else { + val versions = deps.map(_.version).distinct + val versionOpt = mergeVersions(versions) + + versionOpt match { + case Some(version) => \/-(deps.map(dep => dep.copy(version = version))) + case None => -\/(deps) + } + } + } + + val l = m.values.toList + (l.collect{case -\/(dep) => dep}.flatten, l.collect{case \/-(dep) => dep}.flatten) + } + + /** + * If one of our dependency has scope `base`, and a transitive dependency of it has scope `transitive`, + * return the scope of the latter for us, if any. If empty, means the transitive dependency + * should not be considered a dependency for us. + * + * See https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Scope. + */ + def resolveScope(base: Scope, + transitive: Scope): Option[Scope] = + (base, transitive) match { + case (Scope.Compile, other) => Some(other) + case (Scope.Runtime, Scope.Compile) => Some(Scope.Runtime) + case (Scope.Runtime, other) => Some(other) + case _ => None + } + + /** + * Applies `dependencyManagement` to `dependencies`. + * + * Fill empty version / scope / exclusions, for dependencies found in `dependencyManagement`. + */ + def depsWithDependencyManagement(dependencies: Seq[Dependency], + dependencyManagement: Seq[Dependency]): Seq[Dependency] = { + + // See http://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Management + + lazy val m = dependencyManagementAddSeq(Map.empty, dependencyManagement) + + dependencies.map { dep0 => + var dep = dep0 + + for (mgmtDep <- m.get(dependencyManagementKey(dep0))) { + if (dep.version.isEmpty) + dep = dep.copy(version = mgmtDep.version) + if (dep.scope.name.isEmpty) + dep = dep.copy(scope = mgmtDep.scope) + + if (dep.exclusions.isEmpty) + dep = dep.copy(exclusions = mgmtDep.exclusions) + } + + dep + } + } + + + def withDefaultScope(dep: Dependency): Dependency = + if (dep.scope.name.isEmpty) dep.copy(scope = Scope.Compile) + else dep + + /** + * Filters `dependencies` with `exclusions`. + */ + def withExclusions(dependencies: Seq[Dependency], + exclusions: Set[(String, String)]): Seq[Dependency] = { + + val filter = Exclusions(exclusions) + + dependencies + .filter(dep => filter(dep.module.organization, dep.module.name)) + .map(dep => + dep.copy(exclusions = Exclusions.minimize(dep.exclusions ++ exclusions)) + ) + } + + /** + * Get the dependencies of `project`, knowing that it came from dependency `from` (that is, + * `from.module == project.module`). + * + * Substitute properties, update scopes, apply exclusions, and get extra parameters from + * dependency management along the way. + */ + def finalDependencies(from: Dependency, + project: Project): Seq[Dependency] = { + + // Here, we're substituting properties also in dependencies that come from parents + // or dependency management. This may not be the right thing to do. + + val properties = mergeProperties( + project.properties, + Map( + "project.groupId" -> project.module.organization, + "project.artifactId" -> project.module.name, + "project.version" -> project.version + ) + ) + + val deps = + withExclusions( + depsWithDependencyManagement( + // important: properties have to be applied to both, so that dep mgmt can be matched properly + // See the added test with org.ow2.asm:asm-commons:5.0.2 + withProperties(project.dependencies, properties), + withProperties(project.dependencyManagement, properties) + ), + from.exclusions + ) + .map(withDefaultScope) + + deps.flatMap { trDep => + resolveScope(from.scope, trDep.scope) + .map(scope => trDep.copy(scope = scope, optional = trDep.optional || from.optional)) + } + } + + /** + * Default function checking whether a profile is active, given its id, activation conditions, + * and the properties of its project. + */ + def defaultProfileActivation(id: String, + activation: Activation, + props: Map[String, String]): Boolean = { + + if (activation.properties.isEmpty) false + else { + activation.properties.forall { case (name, valueOpt) => + props.get(name).exists{ v => + valueOpt.forall { reqValue => + if (reqValue.startsWith("!")) v != reqValue.drop(1) + else v == reqValue + } + } + } + } + } + + /** + * Default dependency filter used during resolution. + * + * Only follows compile scope / non-optional dependencies. + */ + def defaultFilter(dep: Dependency): Boolean = + !dep.optional && dep.scope == Scope.Compile + +} + + +/** + * State of a dependency resolution. + * + * Done if method `isDone` returns `true`. + * + * @param dependencies: current set of dependencies + * @param conflicts: conflicting dependencies + * @param projectCache: cache of known projects + * @param errorCache: keeps track of the modules whose project definition could not be found + */ +case class Resolution(rootDependencies: Set[Dependency], + dependencies: Set[Dependency], + conflicts: Set[Dependency], + projectCache: Map[Resolution.ModuleVersion, (Repository, Project)], + errorCache: Map[Resolution.ModuleVersion, Seq[String]], + filter: Option[Dependency => Boolean], + profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) { + import Resolution._ + + private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]() + private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized { + finalDependenciesCache.getOrElseUpdate(dep, + projectCache.get(dep.moduleVersion) match { + case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter) + case None => Nil + } + ) + } + + /** + * Transitive dependencies of the current dependencies, according to what there currently is in cache. + * No attempt is made to solve version conflicts here. + */ + def transitiveDependencies: Seq[Dependency] = + (dependencies -- conflicts) + .toList + .flatMap(finalDependencies0) + + /** + * The "next" dependency set, made of the current dependencies and their transitive dependencies, + * trying to solve version conflicts. Transitive dependencies are calculated with the current cache. + * + * May contain dependencies added in previous iterations, but no more required. These are filtered below, see + * `newDependencies`. + * + * Returns a tuple made of the conflicting dependencies, and all the dependencies. + */ + def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = { + merge(rootDependencies.map(withDefaultScope) ++ dependencies ++ transitiveDependencies) + } + + /** + * The modules we miss some info about. + */ + def missingFromCache: Set[ModuleVersion] = { + val modules = dependencies.map(_.moduleVersion) + val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion) + + (modules ++ nextModules) + .filterNot(mod => projectCache.contains(mod) || errorCache.contains(mod)) + } + + + /** + * Whether the resolution is done. + */ + def isDone: Boolean = { + def isFixPoint = { + val (nextConflicts, _) = nextDependenciesAndConflicts + dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet + } + + missingFromCache.isEmpty && isFixPoint + } + + private def eraseVersion(dep: Dependency) = dep.copy(version = "") + + /** + * Returns a map giving the dependencies that brought each of the dependency of the "next" dependency set. + * + * The versions of all the dependencies returned are erased (emptied). + */ + def reverseDependencies: Map[Dependency, Vector[Dependency]] = { + val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts + + val trDepsSeq = + for { + dep <- updatedDeps + trDep <- finalDependencies0(dep) + } yield eraseVersion(trDep) -> eraseVersion(dep) + + val knownDeps = (updatedDeps ++ updatedConflicts).map(eraseVersion).toSet + + trDepsSeq + .groupBy(_._1) + .mapValues(_.map(_._2).toVector) + .filterKeys(knownDeps) + .toList.toMap // Eagerly evaluate filterKeys/mapValues + } + + /** + * Returns dependencies from the "next" dependency set, filtering out + * those that are no more required. + * + * The versions of all the dependencies returned are erased (emptied). + */ + def remainingDependencies: Set[Dependency] = { + val rootDependencies0 = rootDependencies.map(withDefaultScope).map(eraseVersion) + + @tailrec + def helper(reverseDeps: Map[Dependency, Vector[Dependency]]): Map[Dependency, Vector[Dependency]] = { + val (toRemove, remaining) = reverseDeps.partition(kv => kv._2.isEmpty && !rootDependencies0(kv._1)) + + if (toRemove.isEmpty) reverseDeps + else helper(remaining.mapValues(_.filter(x => remaining.contains(x) || rootDependencies0(x))).toList.toMap) + } + + val filteredReverseDependencies = helper(reverseDependencies) + + rootDependencies0 ++ filteredReverseDependencies.keys + } + + /** + * The final next dependency set, stripped of no more required ones. + */ + def newDependencies: Set[Dependency] = { + val remainingDependencies0 = remainingDependencies + nextDependenciesAndConflicts._2 + .filter(dep => remainingDependencies0(eraseVersion(dep))) + .toSet + } + + private def nextNoMissingUnsafe: Resolution = { + val (newConflicts, _) = nextDependenciesAndConflicts + copy(dependencies = newDependencies ++ newConflicts, conflicts = newConflicts.toSet) + } + + /** + * If no module info is missing, the next state of the resolution, which can be immediately calculated. + * Else, the current resolution itself. + */ + def nextIfNoMissing: Resolution = { + val missing = missingFromCache + if (missing.isEmpty) nextNoMissingUnsafe + else this + } + + /** + * Do a new iteration, fetching the missing modules along the way. + */ + def next(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { + val missing = missingFromCache + if (missing.isEmpty) Task.now(nextNoMissingUnsafe) + else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing) + } + + /** + * Required modules for the dependency management of `project`. + */ + def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = { + val approxProperties = + project.parent + .flatMap(projectCache.get) + .map(_._2.properties) + .fold(project.properties)(mergeProperties(project.properties, _)) + + val profileDependencies = + profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) + .flatMap(_.dependencies) + + val modules = + (project.dependencies ++ profileDependencies) + .collect{ case dep if dep.scope == Scope.Import => dep.moduleVersion } ++ + project.parent + + modules.toSet + } + + /** + * Missing modules in cache, to get the full list of dependencies of `project`, taking + * dependency management / inheritance into account. + * + * Note that adding the missing modules to the cache may unveil other missing modules, so + * these modules should be added to the cache, and `dependencyManagementMissing` checked again + * for new missing modules. + */ + def dependencyManagementMissing(project: Project): Set[ModuleVersion] = { + + @tailrec + def helper(toCheck: Set[ModuleVersion], + done: Set[ModuleVersion], + missing: Set[ModuleVersion]): Set[ModuleVersion] = { + + if (toCheck.isEmpty) missing + else if (toCheck.exists(done)) helper(toCheck -- done, done, missing) + else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing) + else if (toCheck.exists(projectCache.contains)) { + val (checking, remaining) = toCheck.partition(projectCache.contains) + val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectCache(mod)._2)) + + helper(remaining ++ directRequirements, done ++ checking, missing) + } else if (toCheck.exists(errorCache.contains)) { + val (errored, remaining) = toCheck.partition(errorCache.contains) + helper(remaining, done ++ errored, missing) + } else + helper(Set.empty, done, missing ++ toCheck) + } + + helper(dependencyManagementRequirements(project), Set(project.moduleVersion), Set.empty) + } + + /** + * Add dependency management / inheritance related items to `project`, from what's available in cache. + * It is recommended to have fetched what `dependencyManagementMissing` returned prior to calling + * `withDependencyManagement`. + */ + def withDependencyManagement(project: Project): Project = { + + val approxProperties = + project.parent + .filter(projectCache.contains) + .map(projectCache(_)._2.properties) + .fold(project.properties)(mergeProperties(project.properties, _)) + + val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) + + val dependencies0 = addDependencies(project.dependencies +: profiles0.map(_.dependencies)) + val properties0 = (project.properties /: profiles0)((acc, p) => mergeProperties(acc, p.properties)) + + val deps = + dependencies0 + .collect{ case dep if dep.scope == Scope.Import && projectCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ + project.parent.filter(projectCache.contains) + val projs = deps.map(projectCache(_)._2) + + val depMgmt = + (project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement))) + .foldLeft(Map.empty[DepMgmtKey, Dependency])(dependencyManagementAddSeq) + + val depsSet = deps.toSet + + project.copy( + dependencies = dependencies0 + .filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++ + project.parent + .filter(projectCache.contains) + .toSeq + .flatMap(projectCache(_)._2.dependencies), + dependencyManagement = depMgmt.values.toSeq, + properties = project.parent + .filter(projectCache.contains) + .map(projectCache(_)._2.properties) + .fold(properties0)(mergeProperties(properties0, _)) + ) + } + + /** + * Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache. + */ + def fetch(modules: Seq[ModuleVersion], + fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { + + val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) + val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) + gatheredLookups.flatMap{ lookupResults => + val errors0 = errorCache ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} + val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj} + + /* + * newProjects are project definitions, fresh from the repositories. We need to add + * dependency management / inheritance-related bits to them. + */ + + newProjects.foldLeft(Task.now(copy(errorCache = errors0))) { case (accTask, (modVer, (repo, proj))) => + for { + current <- accTask + updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule) + proj0 = updated.withDependencyManagement(proj) + } yield updated.copy(projectCache = updated.projectCache + (modVer -> (repo, proj0))) + } + } + } + + def last(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], maxIterations: Int = -1): Task[Resolution] = { + if (maxIterations == 0 || isDone) Task.now(this) + else { + next(fetchModule) + .flatMap(_.last(fetchModule, if (maxIterations > 0) maxIterations - 1 else maxIterations)) + } + } + + def stream(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], run: Task[Resolution] => Resolution): Stream[Resolution] = { + this #:: { + if (isDone) Stream.empty + else run(next(fetchModule)).stream(fetchModule, run) + } + } + + def minDependencies: Set[Dependency] = + Orders.minDependencies(dependencies) + + def artifacts: Seq[Artifact] = + for { + dep <- minDependencies.toSeq + (repo, proj) <- projectCache.get(dep.moduleVersion).toSeq + artifact <- repo.artifacts(dep, proj) + } yield artifact + + def errors: Seq[(Dependency, Seq[String])] = + for { + dep <- dependencies.toSeq + err <- errorCache.get(dep.moduleVersion).toSeq + } yield (dep, err) +} diff --git a/core/src/main/scala/coursier/core/Resolver.scala b/core/src/main/scala/coursier/core/Resolver.scala deleted file mode 100644 index 0c2bd39da..000000000 --- a/core/src/main/scala/coursier/core/Resolver.scala +++ /dev/null @@ -1,677 +0,0 @@ -package coursier.core - -import java.util.regex.Pattern.quote - -import scala.annotation.tailrec -import scala.collection.mutable -import scalaz.concurrent.Task -import scalaz.{EitherT, \/-, \/, -\/} - -object Resolver { - - type ModuleVersion = (Module, String) - - /** - * Try to find `module` among `repositories`. - * - * Look at `repositories` from the left, one-by-one, and stop at first success. - * Else, return all errors, in the same order. - * - * The `version` field of the returned `Project` in case of success may not be - * equal to the provided one, in case the latter is not a specific - * version (e.g. version interval). Which version get chosen depends on - * the repository implementation. - */ - def find(repositories: Seq[Repository], - module: Module, - version: String): EitherT[Task, List[String], (Repository, Project)] = { - - val lookups = repositories.map(repo => repo -> repo.find(module, version).run) - val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[List[String] \/ (Repository, Project)]) { - case (acc, (repo, t)) => - acc.flatMap { - case -\/(errors) => - t.map(res => res - .flatMap(project => - if (project.module == module) \/-((repo, project)) - else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") - ) - .leftMap(error => error :: errors) - ) - - case res @ \/-(_) => - Task.now(res) - } - } - - EitherT(task.map(_.leftMap(_.reverse))).map { case x @ (_, proj) => - assert(proj.module == module) - x - } - } - - /** - * Get the active profiles of `project`, using the current properties `properties`, - * and `profileActivation` stating if a profile is active. - */ - def profiles(project: Project, - properties: Map[String, String], - profileActivation: (String, Activation, Map[String, String]) => Boolean): Seq[Profile] = { - - val activated = project.profiles - .filter(p => profileActivation(p.id, p.activation, properties)) - - def default = project.profiles - .filter(_.activeByDefault.toSeq.contains(true)) - - if (activated.isEmpty) default - else activated - } - - type DepMgmtKey = (String, String, String) - def dependencyManagementKey(dep: Dependency): DepMgmtKey = - dep.artifacts match { - case Artifacts.Maven(type0, _) => (dep.module.organization, dep.module.name, type0) - } - def dependencyManagementAdd(m: Map[DepMgmtKey, Dependency], dep: Dependency): Map[DepMgmtKey, Dependency] = { - val key = dependencyManagementKey(dep) - if (m.contains(key)) m else m + (key -> dep) - } - def dependencyManagementAddSeq(m: Map[DepMgmtKey, Dependency], deps: Seq[Dependency]): Map[DepMgmtKey, Dependency] = - (m /: deps)(dependencyManagementAdd) - - def mergeProperties(m: Map[String, String], other: Map[String, String]): Map[String, String] = { - m ++ other.filterKeys(!m.contains(_)) - } - - def addDependencies(deps: Seq[Seq[Dependency]]): Seq[Dependency] = { - val res = - deps.foldRight((Set.empty[DepMgmtKey], Seq.empty[Dependency])) { - case (deps0, (set, acc)) => - val deps = deps0.filter(dep => !set(dependencyManagementKey(dep))) - (set ++ deps.map(dependencyManagementKey), acc ++ deps) - } - - res._2 - } - - val propRegex = (quote("${") + "([a-zA-Z0-9-.]*)" + quote("}")).r - - /** - * Substitutes `properties` in `dependencies`. - */ - def withProperties(dependencies: Seq[Dependency], - properties: Map[String, String]): Seq[Dependency] = { - - def substituteProps(s: String) = { - val matches = propRegex.findAllMatchIn(s).toList.reverse - if (matches.isEmpty) s - else { - val output = (new StringBuilder(s) /: matches) { - (b, m) => properties.get(m.group(1)).fold(b)(b.replace(m.start, m.end, _)) - } - output.result() - } - } - - dependencies.map{ dep => - dep.copy( - module = dep.module.copy( - organization = substituteProps(dep.module.organization), - name = substituteProps(dep.module.name) - ), - version = substituteProps(dep.version), - artifacts = dep.artifacts match { - case maven: Artifacts.Maven => - maven.copy( - `type` = substituteProps(maven.`type`), - classifier = substituteProps(maven.classifier) - ) - }, - scope = Parse.scope(substituteProps(dep.scope.name)), - exclusions = dep.exclusions - .map{case (org, name) => (substituteProps(org), substituteProps(name))} - // FIXME The content of the optional tag may also be a property in the original POM. - // Maybe not parse it that earlier? - ) - } - } - - /** - * Merge several version constraints together. Returns `None` in case of conflict. - */ - def mergeVersions(versions: Seq[String]): Option[String] = { - val (nonParsedConstraints, parsedConstraints) = - versions - .map(v => v -> Parse.versionConstraint(v)) - .partition(_._2.isEmpty) - - // FIXME Report this in return type, not this way - if (nonParsedConstraints.nonEmpty) - Console.err.println(s"Ignoring unparsed versions: ${nonParsedConstraints.map(_._1)}") - - val constraintOpt = - (Option(VersionInterval.zero) /: parsedConstraints.map(_._2.get.interval)) { - case (acc, itv) => acc.flatMap(_.merge(itv)) - } .map(_.constraint) - - constraintOpt.map(_.repr) - } - - /** - * Merge several dependencies, solving version constraints of duplicated modules. - * Returns the conflicted dependencies, and the (merged) others. - */ - def merge(dependencies: TraversableOnce[Dependency]): (Seq[Dependency], Seq[Dependency]) = { - val m = dependencies - .toList - .groupBy(dep => dep.module) - .mapValues{ deps => - if (deps.lengthCompare(1) == 0) \/-(deps) - else { - val versions = deps.map(_.version).distinct - val versionOpt = mergeVersions(versions) - - versionOpt match { - case Some(version) => \/-(deps.map(dep => dep.copy(version = version))) - case None => -\/(deps) - } - } - } - - val l = m.values.toList - (l.collect{case -\/(dep) => dep}.flatten, l.collect{case \/-(dep) => dep}.flatten) - } - - /** - * If one of our dependency has scope `base`, and a transitive dependency of it has scope `transitive`, - * return the scope of the latter for us, if any. If empty, means the transitive dependency - * should not be considered a dependency for us. - * - * See https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Scope. - */ - def resolveScope(base: Scope, - transitive: Scope): Option[Scope] = - (base, transitive) match { - case (Scope.Compile, other) => Some(other) - case (Scope.Runtime, Scope.Compile) => Some(Scope.Runtime) - case (Scope.Runtime, other) => Some(other) - case _ => None - } - - /** - * Applies `dependencyManagement` to `dependencies`. - * - * Fill empty version / scope / exclusions, for dependencies found in `dependencyManagement`. - */ - def depsWithDependencyManagement(dependencies: Seq[Dependency], - dependencyManagement: Seq[Dependency]): Seq[Dependency] = { - - // See http://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Management - - lazy val m = dependencyManagementAddSeq(Map.empty, dependencyManagement) - - dependencies.map { dep0 => - var dep = dep0 - - for (mgmtDep <- m.get(dependencyManagementKey(dep0))) { - if (dep.version.isEmpty) - dep = dep.copy(version = mgmtDep.version) - if (dep.scope.name.isEmpty) - dep = dep.copy(scope = mgmtDep.scope) - - if (dep.exclusions.isEmpty) - dep = dep.copy(exclusions = mgmtDep.exclusions) - } - - dep - } - } - - /** - * Addition of exclusions. A module is excluded by the result if it is excluded - * by `first`, by `second`, or by both. - */ - def exclusionsAdd(first: Set[(String, String)], - second: Set[(String, String)]): Set[(String, String)] = { - - val (firstAll, firstNonAll) = first.partition{case ("*", "*") => true; case _ => false } - val (secondAll, secondNonAll) = second.partition{case ("*", "*") => true; case _ => false } - - if (firstAll.nonEmpty || secondAll.nonEmpty) Set(("*", "*")) - else { - val firstOrgWildcards = firstNonAll.collect{ case ("*", name) => name } - val firstNameWildcards = firstNonAll.collect{ case (org, "*") => org } - val secondOrgWildcards = secondNonAll.collect{ case ("*", name) => name } - val secondNameWildcards = secondNonAll.collect{ case (org, "*") => org } - - val orgWildcards = firstOrgWildcards ++ secondOrgWildcards - val nameWildcards = firstNameWildcards ++ secondNameWildcards - - val firstRemaining = firstNonAll.filter{ case (org, name) => org != "*" && name != "*" } - val secondRemaining = secondNonAll.filter{ case (org, name) => org != "*" && name != "*" } - - val remaining = (firstRemaining ++ secondRemaining).filterNot{case (org, name) => orgWildcards(name) || nameWildcards(org) } - - orgWildcards.map(name => ("*", name)) ++ nameWildcards.map(org => (org, "*")) ++ remaining - } - } - - def withDefaultScope(dep: Dependency): Dependency = - if (dep.scope.name.isEmpty) dep.copy(scope = Scope.Compile) - else dep - - /** - * Filters `dependencies` with `exclusions`. - */ - def withExclusions(dependencies: Seq[Dependency], - exclusions: Set[(String, String)]): Seq[Dependency] = { - - val (all, notAll) = exclusions.partition{case ("*", "*") => true; case _ => false} - - val orgWildcards = notAll.collect{case ("*", name) => name } - val nameWildcards = notAll.collect{case (org, "*") => org } - - val remaining = notAll.filterNot{case (org, name) => org == "*" || name == "*" } - - dependencies - .filter(dep => - all.isEmpty && - !orgWildcards(dep.module.name) && - !nameWildcards(dep.module.organization) && - !remaining((dep.module.organization, dep.module.name)) - ) - .map(dep => - dep.copy(exclusions = exclusionsAdd(dep.exclusions, exclusions)) - ) - } - - /** - * Get the dependencies of `project`, knowing that it came from dependency `from` (that is, - * `from.module == project.module`). - * - * Substitute properties, update scopes, apply exclusions, and get extra parameters from - * dependency management along the way. - */ - def finalDependencies(from: Dependency, - project: Project): Seq[Dependency] = { - - // Here, we're substituting properties also in dependencies that come from parents - // or dependency management. This may not be the right thing to do. - - val properties = mergeProperties( - project.properties, - Map( - "project.groupId" -> project.module.organization, - "project.artifactId" -> project.module.name, - "project.version" -> project.version - ) - ) - - val deps = - withExclusions( - depsWithDependencyManagement( - // important: properties have to be applied to both, so that dep mgmt can be matched properly - // See the added test with org.ow2.asm:asm-commons:5.0.2 - withProperties(project.dependencies, properties), - withProperties(project.dependencyManagement, properties) - ), - from.exclusions - ) - .map(withDefaultScope) - - deps.flatMap { trDep => - resolveScope(from.scope, trDep.scope) - .map(scope => trDep.copy(scope = scope, optional = trDep.optional || from.optional)) - } - } - - /** - * State of a dependency resolution. - * - * Done if method `isDone` returns `true`. - * - * @param dependencies: current set of dependencies - * @param conflicts: conflicting dependencies - * @param projectsCache: cache of known projects - * @param errors: keeps track of the modules whose project definition could not be found - */ - case class Resolution(rootDependencies: Set[Dependency], - dependencies: Set[Dependency], - conflicts: Set[Dependency], - projectsCache: Map[ModuleVersion, (Repository, Project)], - errors: Map[ModuleVersion, Seq[String]], - filter: Option[Dependency => Boolean], - profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) { - - private val finalDependenciesCache = new mutable.HashMap[Dependency, Seq[Dependency]]() - private def finalDependencies0(dep: Dependency) = finalDependenciesCache.synchronized { - finalDependenciesCache.getOrElseUpdate(dep, - projectsCache.get(dep.moduleVersion) match { - case Some((_, proj)) => finalDependencies(dep, proj).filter(filter getOrElse defaultFilter) - case None => Nil - } - ) - } - - /** - * Transitive dependencies of the current dependencies, according to what there currently is in cache. - * No attempt is made to solve version conflicts here. - */ - def transitiveDependencies: Seq[Dependency] = - for { - dep <- (dependencies -- conflicts).toList - trDep <- finalDependencies0(dep) - } yield trDep - - /** - * The "next" dependency set, made of the current dependencies and their transitive dependencies, - * trying to solve version conflicts. Transitive dependencies are calculated with the current cache. - * - * May contain dependencies added in previous iterations, but no more required. These are filtered below, see - * `newDependencies`. - * - * Returns a tuple made of the conflicting dependencies, and all the dependencies. - */ - def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = { - merge(dependencies ++ transitiveDependencies) - } - - /** - * The modules we miss some info about. - */ - def missingFromCache: Set[ModuleVersion] = { - val modules = dependencies.map(_.moduleVersion) - val nextModules = nextDependenciesAndConflicts._2.map(_.moduleVersion) - - (modules ++ nextModules) - .filterNot(mod => projectsCache.contains(mod) || errors.contains(mod)) - } - - - /** - * Whether the resolution is done. - */ - def isDone: Boolean = { - def isFixPoint = { - val (nextConflicts, _) = nextDependenciesAndConflicts - dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet - } - - missingFromCache.isEmpty && isFixPoint - } - - private def eraseVersion(dep: Dependency) = dep.copy(version = "") - - /** - * Returns a map giving the dependencies that brought each of the dependency of the "next" dependency set. - * - * The versions of all the dependencies returned are erased (emptied). - */ - def reverseDependencies: Map[Dependency, Vector[Dependency]] = { - val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts - - val trDepsSeq = - for { - dep <- updatedDeps - trDep <- finalDependencies0(dep) - } yield eraseVersion(trDep) -> eraseVersion(dep) - - val knownDeps = (updatedDeps ++ updatedConflicts).map(eraseVersion).toSet - - trDepsSeq - .groupBy(_._1) - .mapValues(_.map(_._2).toVector) - .filterKeys(knownDeps) - .toList.toMap // Eagerly evaluate filterKeys/mapValues - } - - /** - * Returns dependencies from the "next" dependency set, filtering out - * those that are no more required. - * - * The versions of all the dependencies returned are erased (emptied). - */ - def remainingDependencies: Set[Dependency] = { - val rootDependencies0 = rootDependencies.map(eraseVersion) - - @tailrec - def helper(reverseDeps: Map[Dependency, Vector[Dependency]]): Map[Dependency, Vector[Dependency]] = { - val (toRemove, remaining) = reverseDeps.partition(kv => kv._2.isEmpty && !rootDependencies0(kv._1)) - - if (toRemove.isEmpty) reverseDeps - else helper(remaining.mapValues(_.filter(x => remaining.contains(x) || rootDependencies0(x))).toList.toMap) - } - - val filteredReverseDependencies = helper(reverseDependencies) - - rootDependencies0 ++ filteredReverseDependencies.keys - } - - /** - * The final next dependency set, stripped of no more required ones. - */ - def newDependencies: Set[Dependency] = { - val remainingDependencies0 = remainingDependencies - nextDependenciesAndConflicts._2 - .filter(dep => remainingDependencies0(eraseVersion(dep))) - .toSet - } - - private def nextNoMissingUnsafe: Resolution = { - val (newConflicts, _) = nextDependenciesAndConflicts - copy(dependencies = newDependencies ++ newConflicts, conflicts = newConflicts.toSet) - } - - /** - * If no module info is missing, the next state of the resolution, which can be immediately calculated. - * Else, the current resolution itself. - */ - def nextIfNoMissing: Resolution = { - val missing = missingFromCache - if (missing.isEmpty) nextNoMissingUnsafe - else this - } - - /** - * Do a new iteration, fetching the missing modules along the way. - */ - def next(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { - val missing = missingFromCache - if (missing.isEmpty) Task.now(nextNoMissingUnsafe) - else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing) - } - - /** - * Required modules for the dependency management of `project`. - */ - def dependencyManagementRequirements(project: Project): Set[ModuleVersion] = { - val approxProperties = - project.parent - .flatMap(projectsCache.get) - .map(_._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) - - val profileDependencies = - profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) - .flatMap(_.dependencies) - - val modules = - (project.dependencies ++ profileDependencies) - .collect{ case dep if dep.scope == Scope.Import => dep.moduleVersion } ++ - project.parent - - modules.toSet - } - - /** - * Missing modules in cache, to get the full list of dependencies of `project`, taking - * dependency management / inheritance into account. - * - * Note that adding the missing modules to the cache may unveil other missing modules, so - * these modules should be added to the cache, and `dependencyManagementMissing` checked again - * for new missing modules. - */ - def dependencyManagementMissing(project: Project): Set[ModuleVersion] = { - - @tailrec - def helper(toCheck: Set[ModuleVersion], - done: Set[ModuleVersion], - missing: Set[ModuleVersion]): Set[ModuleVersion] = { - - if (toCheck.isEmpty) missing - else if (toCheck.exists(done)) helper(toCheck -- done, done, missing) - else if (toCheck.exists(missing)) helper(toCheck -- missing, done, missing) - else if (toCheck.exists(projectsCache.contains)) { - val (checking, remaining) = toCheck.partition(projectsCache.contains) - val directRequirements = checking.flatMap(mod => dependencyManagementRequirements(projectsCache(mod)._2)) - - helper(remaining ++ directRequirements, done ++ checking, missing) - } else if (toCheck.exists(errors.contains)) { - val (errored, remaining) = toCheck.partition(errors.contains) - helper(remaining, done ++ errored, missing) - } else - helper(Set.empty, done, missing ++ toCheck) - } - - helper(dependencyManagementRequirements(project), Set(project.moduleVersion), Set.empty) - } - - /** - * Add dependency management / inheritance related items to `project`, from what's available in cache. - * It is recommended to have fetched what `dependencyManagementMissing` returned prior to calling - * `withDependencyManagement`. - */ - def withDependencyManagement(project: Project): Project = { - - val approxProperties = - project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) - - val profiles0 = profiles(project, approxProperties, profileActivation getOrElse defaultProfileActivation) - - val dependencies0 = addDependencies(project.dependencies +: profiles0.map(_.dependencies)) - val properties0 = (project.properties /: profiles0)((acc, p) => mergeProperties(acc, p.properties)) - - val deps = - dependencies0 - .collect{ case dep if dep.scope == Scope.Import && projectsCache.contains(dep.moduleVersion) => dep.moduleVersion } ++ - project.parent.filter(projectsCache.contains) - val projs = deps.map(projectsCache(_)._2) - - val depMgmt = - (project.dependencyManagement +: (profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement))) - .foldLeft(Map.empty[DepMgmtKey, Dependency])(dependencyManagementAddSeq) - - val depsSet = deps.toSet - - project.copy( - dependencies = dependencies0 - .filterNot(dep => dep.scope == Scope.Import && depsSet(dep.moduleVersion)) ++ - project.parent - .filter(projectsCache.contains) - .toSeq - .flatMap(projectsCache(_)._2.dependencies), - dependencyManagement = depMgmt.values.toSeq, - properties = project.parent - .filter(projectsCache.contains) - .map(projectsCache(_)._2.properties) - .fold(properties0)(mergeProperties(properties0, _)) - ) - } - - /** - * Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache. - */ - def fetch(modules: Seq[ModuleVersion], - fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { - - val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) - val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) - gatheredLookups.flatMap{ lookupResults => - val errors0 = errors ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} - val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj} - - /* - * newProjects are project definitions, fresh from the repositories. We need to add - * dependency management / inheritance-related bits to them. - */ - - newProjects.foldLeft(Task.now(copy(errors = errors0))) { case (accTask, (modVer, (repo, proj))) => - for { - current <- accTask - updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule) - proj0 = updated.withDependencyManagement(proj) - } yield updated.copy(projectsCache = updated.projectsCache + (modVer -> (repo, proj0))) - } - } - } - - } - - /** - * Default function checking whether a profile is active, given its id, activation conditions, - * and the properties of its project. - */ - def defaultProfileActivation(id: String, - activation: Activation, - props: Map[String, String]): Boolean = { - - if (activation.properties.isEmpty) false - else { - activation.properties.forall { case (name, valueOpt) => - props.get(name).exists{ v => - valueOpt.forall { reqValue => - if (reqValue.startsWith("!")) v != reqValue.drop(1) - else v == reqValue - } - } - } - } - } - - /** - * Default dependency filter used during resolution. - * - * Only follows compile scope / non-optional dependencies. - */ - def defaultFilter(dep: Dependency): Boolean = - !dep.optional && dep.scope == Scope.Compile - - /** - * Get all the transitive dependencies of `dependencies`, solving any dependency version mismatch. - * - * Iteratively fetches the missing info of the current dependencies / add newly discovered dependencies - * to the current ones. The maximum number of such iterations can be bounded with `maxIterations`. - * - * ... - * - */ - def resolve(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], - maxIterations: Option[Int], - filter: Option[Dependency => Boolean], - profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]): Task[Resolution] = { - - val dependencies0 = dependencies.map(withDefaultScope) - - val startResolution = Resolution( - dependencies0, dependencies0, Set.empty, - Map.empty, Map.empty, - filter, - profileActivation - ) - - def helper(resolution: Resolution, remainingIter: Option[Int]): Task[(Resolution, Option[Int])] = { - if (resolution.isDone || remainingIter.exists(_ <= 0)) - Task.now((resolution, remainingIter)) - else - resolution.next(fetch).flatMap(helper(_, remainingIter.map(_ - 1))) - } - - helper(startResolution, maxIterations).map(_._1) - } - -} diff --git a/core/src/main/scala/coursier/core/Xml.scala b/core/src/main/scala/coursier/core/Xml.scala index 621b3272f..caa2a3156 100644 --- a/core/src/main/scala/coursier/core/Xml.scala +++ b/core/src/main/scala/coursier/core/Xml.scala @@ -85,7 +85,7 @@ object Xml { mod, version0, scopeOpt getOrElse defaultScope, - Artifacts.Maven(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), + Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), exclusions.map(mod => (mod.organization, mod.name)).toSet, optional ) diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 01a2083b5..2cfd39143 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -1,6 +1,9 @@ import scalaz.EitherT import scalaz.concurrent.Task +/** + * Pulls definitions from coursier.core, with default arguments. + */ package object coursier { type Dependency = core.Dependency @@ -8,20 +11,17 @@ package object coursier { def apply(module: Module, version: String, scope: Scope = Scope.Other(""), // Substituted by Resolver with its own default scope (compile) - artifacts: Artifacts = Artifacts.Maven(), + attributes: Attributes = Attributes(), exclusions: Set[(String, String)] = Set.empty, optional: Boolean = false): Dependency = - core.Dependency(module, version, scope, artifacts, exclusions, optional) + core.Dependency(module, version, scope, attributes, exclusions, optional) } - type Artifacts = core.Artifacts - object Artifacts { - type Maven = core.Artifacts.Maven - object Maven { - def apply(`type`: String = "jar", - classifier: String = ""): Maven = - core.Artifacts.Maven(`type`, classifier) - } + type Attributes = core.Attributes + object Attributes { + def apply(`type`: String = "jar", + classifier: String = ""): Attributes = + core.Attributes(`type`, classifier) } type Project = core.Project @@ -68,19 +68,19 @@ package object coursier { type Repository = core.Repository def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, List[String], (Repository, Project)] = - modVersion => core.Resolver.find(repositories, modVersion._1, modVersion._2) + modVersion => core.Resolution.find(repositories, modVersion._1, modVersion._2) - type Resolution = core.Resolver.Resolution + type Resolution = core.Resolution object Resolution { val empty = apply() def apply(rootDependencies: Set[Dependency] = Set.empty, dependencies: Set[Dependency] = Set.empty, conflicts: Set[Dependency] = Set.empty, - projectsCache: Map[ModuleVersion, (Repository, Project)] = Map.empty, - errors: Map[ModuleVersion, Seq[String]] = Map.empty, + projectCache: Map[ModuleVersion, (Repository, Project)] = Map.empty, + errorCache: Map[ModuleVersion, Seq[String]] = Map.empty, filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution = - core.Resolver.Resolution(rootDependencies, dependencies, conflicts, projectsCache, errors, filter, profileActivation) + core.Resolution(rootDependencies, dependencies, conflicts, projectCache, errorCache, filter, profileActivation) } def resolve(dependencies: Set[Dependency], @@ -88,6 +88,26 @@ package object coursier { maxIterations: Option[Int] = Some(200), filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { - core.Resolver.resolve(dependencies, fetch, maxIterations, filter, profileActivation) + + val startResolution = Resolution( + dependencies, Set.empty, Set.empty, + Map.empty, Map.empty, + filter, + profileActivation + ) + + startResolution.last(fetch, maxIterations.getOrElse(-1)) } + + type Artifact = core.Artifact + object Artifact { + def apply(url: String, + extra: Map[String, String] = Map.empty, + attributes: Attributes = Attributes()): Artifact = + core.Artifact(url, extra, attributes) + } + + type MavenRepository[G <: core.FetchMetadata] = core.MavenRepository[G] + val MavenRepository: core.MavenRepository.type = core.MavenRepository + } diff --git a/core/src/main/scala/coursier/repository/package.scala b/core/src/main/scala/coursier/repository/package.scala new file mode 100644 index 000000000..c448f25c5 --- /dev/null +++ b/core/src/main/scala/coursier/repository/package.scala @@ -0,0 +1,12 @@ +package coursier + +import coursier.core.DefaultFetchMetadata + +package object repository { + + val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/")) + + val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) + val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) + +} diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 1d58286d6..f8348043f 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -12,12 +12,8 @@ object CentralTests extends TestSuite { repository.mavenCentral ) - def repr(dep: Dependency) = { - val (type0, classifier) = dep.artifacts match { - case maven: Artifacts.Maven => (maven.`type`, maven.classifier) - } - s"${dep.module.organization}:${dep.module.name}:$type0:${Some(classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" - } + def repr(dep: Dependency) = + s"${dep.module.organization}:${dep.module.name}:${dep.attributes.`type`}:${Some(dep.attributes.classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" def resolutionCheck(module: Module, version: String) = async { @@ -39,10 +35,10 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("ch.qos.logback", "logback-classic"), "1.1.3") val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) - .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here + .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set( dep.withCompileScope, Dependency(Module("ch.qos.logback", "logback-core"), "1.1.3").withCompileScope, @@ -55,10 +51,10 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("org.ow2.asm", "asm-commons"), "5.0.2") val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) - .copy(projectsCache = Map.empty, errors = Map.empty) // No validating these here + .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set( dep.withCompileScope, Dependency(Module("org.ow2.asm", "asm-tree"), "5.0.2").withCompileScope, @@ -71,17 +67,17 @@ object CentralTests extends TestSuite { async { val dep = Dependency(Module("joda-time", "joda-time"), "[2.2,2.8]") val res0 = await(resolve(Set(dep), fetchFrom(repositories)).runF) - val res = res0.copy(projectsCache = Map.empty, errors = Map.empty) + val res = res0.copy(projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set( dep.withCompileScope)) assert(res == expected) - assert(res0.projectsCache.contains(dep.moduleVersion)) + assert(res0.projectCache.contains(dep.moduleVersion)) - val (_, proj) = res0.projectsCache(dep.moduleVersion) + val (_, proj) = res0.projectCache(dep.moduleVersion) assert(proj.version == "2.8") } } diff --git a/core/src/test/scala/coursier/test/ExclusionsTests.scala b/core/src/test/scala/coursier/test/ExclusionsTests.scala index 3eb775d95..051421384 100644 --- a/core/src/test/scala/coursier/test/ExclusionsTests.scala +++ b/core/src/test/scala/coursier/test/ExclusionsTests.scala @@ -2,10 +2,12 @@ package coursier package test import utest._ -import core.Resolver.exclusionsAdd object ExclusionsTests extends TestSuite { + def exclusionsAdd(e1: Set[(String, String)], e2: Set[(String, String)]) = + core.Exclusions.minimize(e1 ++ e2) + val tests = TestSuite { val e1 = Set(("org1", "name1")) val e2 = Set(("org2", "name2")) diff --git a/core/src/test/scala/coursier/test/PomParsingTests.scala b/core/src/test/scala/coursier/test/PomParsingTests.scala index 83d339acf..3135c4905 100644 --- a/core/src/test/scala/coursier/test/PomParsingTests.scala +++ b/core/src/test/scala/coursier/test/PomParsingTests.scala @@ -21,7 +21,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Dependency(Module("comp", "lib"), "2.1", artifacts = Artifacts.Maven(classifier = "extra"))) + val expected = \/-(Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra"))) val result = Xml.dependency(xmlParse(depNode).right.get) diff --git a/core/src/test/scala/coursier/test/ResolverTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala similarity index 88% rename from core/src/test/scala/coursier/test/ResolverTests.scala rename to core/src/test/scala/coursier/test/ResolutionTests.scala index 58f1457c6..4734bd0f5 100644 --- a/core/src/test/scala/coursier/test/ResolverTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -1,13 +1,12 @@ package coursier package test -import coursier.core.Resolver import utest._ import scala.async.Async.{async, await} import coursier.test.compatibility._ -object ResolverTests extends TestSuite { +object ResolutionTests extends TestSuite { implicit class ProjectOps(val p: Project) extends AnyVal { def kv: (ModuleVersion, (Repository, Project)) = p.moduleVersion -> (testRepository, p) @@ -167,9 +166,9 @@ object ResolverTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - errors = Map(dep.moduleVersion -> Seq("Not found")) + errorCache = Map(dep.moduleVersion -> Seq("Not found")) ) assert(res == expected) @@ -184,9 +183,9 @@ object ResolverTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - projectsCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) + projectCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) ) assert(res == expected) @@ -202,9 +201,9 @@ object ResolverTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope, trDep.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv, projectsMap(trDep.moduleVersion).kv ) @@ -226,9 +225,9 @@ object ResolverTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -251,9 +250,9 @@ object ResolverTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -276,9 +275,9 @@ object ResolverTests extends TestSuite { ).runF) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) ) @@ -296,9 +295,9 @@ object ResolverTests extends TestSuite { ).runF).copy(filter = None) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - projectsCache = Map( + projectCache = Map( projectsMap(dep.moduleVersion).kv ) ) @@ -317,10 +316,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -337,10 +336,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -356,10 +355,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -373,10 +372,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ) @@ -392,10 +391,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -413,10 +412,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -436,10 +435,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -458,10 +457,10 @@ object ResolverTests extends TestSuite { Set(dep), fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = Set(dep.withCompileScope), + rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) @@ -482,10 +481,10 @@ object ResolverTests extends TestSuite { deps, fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectsCache = Map.empty, errors = Map.empty) + ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( - rootDependencies = deps.map(_.withCompileScope), + rootDependencies = deps, dependencies = (deps ++ trDeps).map(_.withCompileScope) ) @@ -496,7 +495,7 @@ object ResolverTests extends TestSuite { 'parts{ 'propertySubstitution{ val res = - Resolver.withProperties( + core.Resolution.withProperties( Seq(Dependency(Module("a-company", "a-name"), "${a.property}")), Map("a.property" -> "a-version")) val expected = Seq(Dependency(Module("a-company", "a-name"), "a-version")) diff --git a/core/src/test/scala/coursier/test/TestRepository.scala b/core/src/test/scala/coursier/test/TestRepository.scala index b70077b34..fcdf892ff 100644 --- a/core/src/test/scala/coursier/test/TestRepository.scala +++ b/core/src/test/scala/coursier/test/TestRepository.scala @@ -12,8 +12,5 @@ class TestRepository(projects: Map[(Module, String), Project]) extends Repositor EitherT(Task.now( projects.get((module, version)).toRightDisjunction("Not found") )) - def versions(organization: String, name: String, cachePolicy: CachePolicy) = - EitherT(Task.now[String \/ Versions]( - -\/("Not supported") - )) + def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = ??? } diff --git a/core/src/test/scala/coursier/test/package.scala b/core/src/test/scala/coursier/test/package.scala index 863eb2519..af3c811e8 100644 --- a/core/src/test/scala/coursier/test/package.scala +++ b/core/src/test/scala/coursier/test/package.scala @@ -1,9 +1,26 @@ package coursier +import scalaz.EitherT +import scalaz.concurrent.Task + package object test { implicit class DependencyOps(val underlying: Dependency) extends AnyVal { def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile) } + def resolve(dependencies: Set[Dependency], + fetch: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], + maxIterations: Option[Int] = Some(200), + filter: Option[Dependency => Boolean] = None, + profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { + + val startResolution = Resolution( + dependencies, + filter = filter, + profileActivation = profileActivation + ) + + startResolution.last(fetch, maxIterations.getOrElse(-1)) + } } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala new file mode 100644 index 000000000..014c56ea9 --- /dev/null +++ b/files/src/main/scala/coursier/Files.scala @@ -0,0 +1,121 @@ +package coursier + +import java.net.URL + +import coursier.core.CachePolicy + +import scala.annotation.tailrec +import scalaz.{-\/, \/-, \/, EitherT} +import scalaz.concurrent.Task + +import java.io._ + +// FIXME This kind of side-effecting API is lame, we should aim at a more functional one. +trait FilesLogger { + def foundLocally(f: File): Unit + def downloadingArtifact(url: String): Unit + def downloadedArtifact(url: String, success: Boolean): Unit +} + +case class Files(cache: Seq[(String, File)], + tmp: () => File, + logger: Option[FilesLogger] = None) { + + def file(artifact: Artifact, + cachePolicy: CachePolicy): EitherT[Task, String, File] = { + + cache.find{case (base, _) => artifact.url.startsWith(base)} match { + case None => ??? + case Some((base, cacheDir)) => + val file = new File(cacheDir, artifact.url.stripPrefix(base)) + + def locally = { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(file)) + \/-(file) + } + else -\/("Not found in cache") + } + } + + def remote = { + // FIXME A lot of things can go wrong here and are not properly handled: + // - checksums should be validated + // - what if the connection gets closed during the transfer (partial file on disk)? + // - what if someone is trying to write this file at the same time? (no locking of any kind yet) + // - ... + + Task { + try { + file.getParentFile.mkdirs() + + logger.foreach(_.downloadingArtifact(artifact.url)) + + val url = new URL(artifact.url) + val b = Array.fill[Byte](Files.bufferSize)(0) + val in = new BufferedInputStream(url.openStream(), Files.bufferSize) + + try { + val out = new FileOutputStream(file) + try { + @tailrec + def helper(): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + helper() + } + } + + helper() + } finally out.close() + } finally in.close() + + logger.foreach(_.downloadedArtifact(artifact.url, success = true)) + \/-(file) + } + catch { case e: Exception => + logger.foreach(_.downloadedArtifact(artifact.url, success = false)) + -\/(e.getMessage) + } + } + } + + EitherT(cachePolicy(locally)(remote)) + } + } + +} + +object Files { + + var bufferSize = 1024*1024 + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap(_.getMessage) + } + +} diff --git a/project/Coursier.scala b/project/Coursier.scala index 60e22572e..d11e0f0c7 100644 --- a/project/Coursier.scala +++ b/project/Coursier.scala @@ -101,8 +101,20 @@ object CoursierBuild extends Build { ) .enablePlugins(ScalaJSPlugin) - lazy val cli = Project(id = "cli", base = file("cli")) + lazy val files = Project(id = "files", base = file("files")) .dependsOn(coreJvm) + .settings(commonSettings: _*) + .settings( + name := "coursier-files", + libraryDependencies ++= Seq( + "org.http4s" %% "http4s-blazeclient" % "0.8.2", + "com.lihaoyi" %% "utest" % "0.3.0" % "test" + ), + testFrameworks += new TestFramework("utest.runner.Framework") + ) + + lazy val cli = Project(id = "cli", base = file("cli")) + .dependsOn(coreJvm, files) .settings(commonSettings ++ packAutoSettings ++ publishPackTxzArchive ++ publishPackZipArchive: _*) .settings( packArchivePrefix := s"coursier-cli_${scalaBinaryVersion.value}", diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 1b5c73f57..528e4cd04 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,7 +1,7 @@ package coursier package web -import coursier.core.{Resolver, Logger, Remote} +import coursier.core.{DefaultFetchMetadata, Logger} import japgolly.scalajs.react.vdom.{TagMod, Attr} import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope} @@ -18,7 +18,7 @@ case class ResolutionOptions(followOptional: Boolean = false, keepTest: Boolean = false) case class State(modules: Seq[Dependency], - repositories: Seq[Remote], + repositories: Seq[MavenRepository[DefaultFetchMetadata]], options: ResolutionOptions, resolutionOpt: Option[Resolution], editModuleIdx: Int, @@ -71,13 +71,15 @@ class Backend($: BackendScope[Unit, State]) { def updateTree(resolution: Resolution, target: String, reverse: Boolean) = { def depsOf(dep: Dependency) = - resolution.projectsCache.get(dep.moduleVersion).toSeq.flatMap(t => Resolver.finalDependencies(dep, t._2).filter(resolution.filter getOrElse Resolver.defaultFilter)) + resolution.projectCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter)) + + val minDependencies = resolution.minDependencies lazy val reverseDeps = { var m = Map.empty[Module, Seq[Dependency]] for { - dep <- resolution.dependencies + dep <- minDependencies trDep <- depsOf(dep) } { m += trDep.module -> (m.getOrElse(trDep.module, Nil) :+ dep) @@ -95,8 +97,8 @@ class Backend($: BackendScope[Unit, State]) { else Seq("nodes" -> js.Array(deps.map(tree): _*)) }: _*) - println(resolution.dependencies.toList.map(tree).map(js.JSON.stringify(_))) - g.$(target).treeview(js.Dictionary("data" -> js.Array(resolution.dependencies.toList.map(tree): _*))) + println(minDependencies.toList.map(tree).map(js.JSON.stringify(_))) + g.$(target).treeview(js.Dictionary("data" -> js.Array(minDependencies.toList.map(tree): _*))) } def resolve(action: => Unit = ()) = { @@ -119,11 +121,14 @@ class Backend($: BackendScope[Unit, State]) { } val s = $.state - def task = coursier.resolve( - s.modules.toSet, - fetchFrom(s.repositories.map(_.copy(logger = Some(logger)))), - filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) - ) + def task = { + val res = coursier.Resolution( + s.modules.toSet, + filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) + ) + + res.last(fetchFrom(s.repositories.map(r => r.copy(fetchMetadata = r.fetchMetadata.copy(logger = Some(logger))))), 100) + } // For reasons that are unclear to me, not delaying this when using the runNow execution context // somehow discards the $.modState above. (Not a major problem as queue is used by default.) @@ -218,26 +223,22 @@ object App { ) def depItem(dep: Dependency, finalVersionOpt: Option[String]) = { - val (type0, classifier) = dep.artifacts match { - case maven: Artifacts.Maven => (maven.`type`, maven.classifier) - } - <.tr( - ^.`class` := (if (res.errors.contains(dep.moduleVersion)) "danger" else ""), + ^.`class` := (if (res.errorCache.contains(dep.moduleVersion)) "danger" else ""), <.td(dep.module.organization), <.td(dep.module.name), <.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")), <.td(Seq[Seq[TagMod]]( if (dep.scope == Scope.Compile) Seq() else Seq(infoLabel(dep.scope.name)), - if (type0.isEmpty || type0 == "jar") Seq() else Seq(infoLabel(type0)), - if (classifier.isEmpty) Seq() else Seq(infoLabel(classifier)), + if (dep.attributes.`type`.isEmpty || dep.attributes.`type` == "jar") Seq() else Seq(infoLabel(dep.attributes.`type`)), + if (dep.attributes.classifier.isEmpty) Seq() else Seq(infoLabel(dep.attributes.classifier)), Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq, if (dep.optional) Seq(infoLabel("optional")) else Seq(), - res.errors.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq + res.errorCache.get(dep.moduleVersion).map(errs => errorPopOver("Error", errs.mkString("; "))).toSeq )), <.td(Seq[Seq[TagMod]]( - res.projectsCache.get(dep.moduleVersion) match { - case Some((repo: Remote, _)) => + res.projectCache.get(dep.moduleVersion) match { + case Some((MavenRepository(fetchMetadata, _), _)) => // FIXME Maven specific, generalize if/when adding support for Ivy val version0 = finalVersionOpt getOrElse dep.version val relPath = @@ -248,10 +249,10 @@ object App { ) Seq( - <.a(^.href := s"${repo.base}${relPath.mkString("/")}.pom", + <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.pom", <.span(^.`class` := "label label-info", "POM") ), - <.a(^.href := s"${repo.base}${relPath.mkString("/")}.jar", + <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.jar", <.span(^.`class` := "label label-info", "JAR") ) ) @@ -262,7 +263,7 @@ object App { ) } - val sortedDeps = res.dependencies.toList + val sortedDeps = res.minDependencies.toList .sortBy(dep => coursier.core.Module.unapply(dep.module).get) <.table(^.`class` := "table", @@ -276,7 +277,7 @@ object App { ) ), <.tbody( - sortedDeps.map(dep => depItem(dep, res.projectsCache.get(dep.moduleVersion).map(_._2.version).filter(_ != dep.version))) + sortedDeps.map(dep => depItem(dep, res.projectCache.get(dep.moduleVersion).map(_._2.version).filter(_ != dep.version))) ) ) } @@ -385,19 +386,19 @@ object App { val modules = dependenciesTable("Dependencies") - val repositories = ReactComponentB[Seq[Remote]]("Repositories") + val repositories = ReactComponentB[Seq[MavenRepository[DefaultFetchMetadata]]]("Repositories") .render{ repos => - def repoItem(repo: Remote) = + def repoItem(repo: MavenRepository[DefaultFetchMetadata]) = <.tr( <.td( - <.a(^.href := repo.base, - repo.base + <.a(^.href := repo.fetchMetadata.root, + repo.fetchMetadata.root ) ) ) val sortedRepos = repos - .sortBy(repo => repo.base) + .sortBy(repo => repo.fetchMetadata.root) <.table(^.`class` := "table", <.thead(