From 95f51fdb8f0e5a71df2d89953c59ac33df1cb0c7 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 00:59:07 +0100 Subject: [PATCH 01/11] Local ivy2 support --- .../main/scala/coursier/cli/Coursier.scala | 19 +++- .../coursier/core/DefaultFetchMetadata.scala | 67 +++++------ .../main/scala/coursier/core/Repository.scala | 83 ++++++++++---- .../scala/coursier/repository/package.scala | 2 + files/src/main/scala/coursier/Files.scala | 104 ++++++++++-------- 5 files changed, 168 insertions(+), 107 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 3f06b9891..b1cbef11b 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -22,9 +22,7 @@ case class Coursier(scope: List[String], val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/metadata/central") val centralFilesCacheDir = new File(sys.props("user.home") + "/.coursier/cache/files/central") - val base = centralCacheDir.toURI - def fileRepr(f: File) = - base.relativize(f.toURI).getPath + def fileRepr(f: File) = f.toString val logger: MetadataFetchLogger with FilesLogger = new MetadataFetchLogger with FilesLogger { def println(s: String) = Console.err.println(s) @@ -60,7 +58,12 @@ case class Coursier(scope: List[String], ) ) val repositories = Seq[Repository]( - cachedMavenCentral + cachedMavenCentral, + repository.ivy2Local.copy( + fetchMetadata = repository.ivy2Local.fetchMetadata.copy( + logger = Some(logger) + ) + ) ) val (splitDependencies, malformed) = remainingArgs.toList @@ -132,7 +135,13 @@ case class Coursier(scope: List[String], val artifacts = res.artifacts - val files = new Files(Seq(cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir), () => ???, Some(logger)) + val files = new Files( + Seq( + cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir + ), + () => ???, + Some(logger) + ) val tasks = artifacts.map(files.file(_, cachePolicy).run) val task = Task.gatherUnordered(tasks) diff --git a/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala index 30f7f87af..70bd9dea9 100644 --- a/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala +++ b/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala @@ -2,7 +2,7 @@ package coursier package core import java.io._ -import java.net.URL +import java.net.{URI, URL} import scala.io.Codec import scalaz._, Scalaz._ @@ -19,18 +19,13 @@ case class DefaultFetchMetadata(root: String, cache: Option[File] = None, logger: Option[MetadataFetchLogger] = None) extends FetchMetadata { - def apply(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, String] = { - lazy val localFile = { - for { - cache0 <- cache.toRightDisjunction("No cache") - f = new File(cache0, artifact.url) - } yield f - } + val isLocal = root.startsWith("file:///") - def locally = { + def apply(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, String] = { + def locally(eitherFile: String \/ File) = { Task { for { - f0 <- localFile + f0 <- eitherFile f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") content <- \/.fromTryCatchNonFatal{ logger.foreach(_.readingFromCache(f)) @@ -40,32 +35,42 @@ case class DefaultFetchMetadata(root: String, } } - def remote = { - val urlStr = root + artifact.url - val url = new URL(urlStr) + if (isLocal) EitherT(locally(\/-(new File(new URI(root + artifact.url) .getPath)))) + else { + lazy val localFile = { + for { + cache0 <- cache.toRightDisjunction("No cache") + f = new File(cache0, artifact.url) + } yield f + } - def log = Task(logger.foreach(_.downloading(urlStr))) - def get = DefaultFetchMetadata.readFully(url.openStream()) + def remote = { + val urlStr = root + artifact.url + val url = new URL(urlStr) - log.flatMap(_ => get) - } + def log = Task(logger.foreach(_.downloading(urlStr))) + def get = DefaultFetchMetadata.readFully(url.openStream()) - def save(s: String) = { - localFile.fold(_ => Task.now(()), f => - Task { - if (!f.exists()) { - logger.foreach(_.puttingInCache(f)) - f.getParentFile.mkdirs() - val w = new PrintWriter(f) - try w.write(s) - finally w.close() - () + log.flatMap(_ => get) + } + + def save(s: String) = { + localFile.fold(_ => Task.now(()), f => + Task { + if (!f.exists()) { + logger.foreach(_.puttingInCache(f)) + f.getParentFile.mkdirs() + val w = new PrintWriter(f) + try w.write(s) + finally w.close() + () + } } - } - ) - } + ) + } - EitherT(cachePolicy.saving(locally)(remote)(save)) + EitherT(cachePolicy.saving(locally(localFile))(remote)(save)) + } } } diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index d95d32eef..a8bb44f7d 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -66,7 +66,7 @@ object Repository { Artifact.javadoc -> (base + "-javadoc.jar"), Artifact.javadocSigMd5 -> (base + "-javadoc.jar.asc.md5"), Artifact.javadocSigSha1 -> (base + "-javadoc.jar.asc.sha1"), - Artifact.javadocSig -> (base + "-javadoc.asc.jar") + Artifact.javadocSig -> (base + "-javadoc.jar.asc") )) } } @@ -83,27 +83,35 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, import Repository._ + def ivyLikePath(org: String, name: String, version: String, subDir: String, baseSuffix: String, ext: String) = Seq( + org, + name, + version, + subDir, + s"$name$baseSuffix.$ext" + ) + def projectArtifact(module: Module, version: String): Artifact = { - if (ivyLike) ??? - else { - val path = ( + val path = ( + if (ivyLike) + ivyLikePath(module.organization, module.name, version, "poms", "", "pom") + else module.organization.split('.').toSeq ++ Seq( module.name, version, s"${module.name}-$version.pom" ) - ) .map(encodeURIComponent) + ) .map(encodeURIComponent) - Artifact( - path.mkString("/"), - Map( - Artifact.md5 -> "", - Artifact.sha1 -> "" - ), - Attributes("pom", "") - ) - .withDefaultSignature - } + Artifact( + path.mkString("/"), + Map( + Artifact.md5 -> "", + Artifact.sha1 -> "" + ), + Attributes("pom", "") + ) + .withDefaultSignature } def versionsArtifact(module: Module): Option[Artifact] = @@ -202,12 +210,18 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = { + def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = + ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext) + val path = - dependency.module.organization.split('.').toSeq ++ Seq( - dependency.module.name, - project.version, - s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" - ) + if (ivyLike) + ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) + else + dependency.module.organization.split('.').toSeq ++ Seq( + dependency.module.name, + project.version, + s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + ) var artifact = Artifact( @@ -217,10 +231,31 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, ) .withDefaultChecksums - if (dependency.attributes.`type` == "jar") - artifact = artifact - .withDefaultSignature - .withJavadocSources + if (dependency.attributes.`type` == "jar") { + artifact = artifact.withDefaultSignature + + artifact = + if (ivyLike) { + val srcPath = fetchMetadata.root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") + val javadocPath = fetchMetadata.root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") + + artifact + .copy(extra = artifact.extra ++ Map( + Artifact.sourcesMd5 -> (srcPath + ".md5"), + Artifact.sourcesSha1 -> (srcPath + ".sha1"), + Artifact.sources -> srcPath, + Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), + Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), + Artifact.sourcesSig -> (srcPath + ".asc"), + Artifact.javadocMd5 -> (javadocPath + ".md5"), + Artifact.javadocSha1 -> (javadocPath + ".sha1"), + Artifact.javadoc -> javadocPath, + Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), + Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), + Artifact.javadocSig -> (javadocPath + ".asc") + )) + } else artifact.withJavadocSources + } Seq(artifact) } diff --git a/core/src/main/scala/coursier/repository/package.scala b/core/src/main/scala/coursier/repository/package.scala index c448f25c5..93acd2c09 100644 --- a/core/src/main/scala/coursier/repository/package.scala +++ b/core/src/main/scala/coursier/repository/package.scala @@ -9,4 +9,6 @@ package object repository { val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) + lazy val ivy2Local = MavenRepository(DefaultFetchMetadata("file://" + sys.props("user.home") + "/.ivy2/local/"), ivyLike = true) + } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index 014c56ea9..9d847c27e 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -1,6 +1,6 @@ package coursier -import java.net.URL +import java.net.{URI, URL} import coursier.core.CachePolicy @@ -24,65 +24,75 @@ case class Files(cache: Seq[(String, File)], def file(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, File] = { - cache.find{case (base, _) => artifact.url.startsWith(base)} match { - case None => ??? - case Some((base, cacheDir)) => - val file = new File(cacheDir, artifact.url.stripPrefix(base)) + if (artifact.url.startsWith("file:///")) { + val f = new File(new URI(artifact.url) .getPath) + EitherT(Task.now( + if (f.exists()) { + logger.foreach(_.foundLocally(f)) + \/-(f) + } else -\/("Not found") + )) + } else { + cache.find{case (base, _) => artifact.url.startsWith(base)} match { + case None => ??? + case Some((base, cacheDir)) => + val file = new File(cacheDir, artifact.url.stripPrefix(base)) - def locally = { - Task { - if (file.exists()) { - logger.foreach(_.foundLocally(file)) - \/-(file) + def locally = { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(file)) + \/-(file) + } + else -\/("Not found in cache") } - else -\/("Not found in cache") } - } - def remote = { - // FIXME A lot of things can go wrong here and are not properly handled: - // - checksums should be validated - // - what if the connection gets closed during the transfer (partial file on disk)? - // - what if someone is trying to write this file at the same time? (no locking of any kind yet) - // - ... - - Task { - try { - file.getParentFile.mkdirs() - - logger.foreach(_.downloadingArtifact(artifact.url)) - - val url = new URL(artifact.url) - val b = Array.fill[Byte](Files.bufferSize)(0) - val in = new BufferedInputStream(url.openStream(), Files.bufferSize) + def remote = { + // FIXME A lot of things can go wrong here and are not properly handled: + // - checksums should be validated + // - what if the connection gets closed during the transfer (partial file on disk)? + // - what if someone is trying to write this file at the same time? (no locking of any kind yet) + // - ... + Task { try { - val out = new FileOutputStream(file) + file.getParentFile.mkdirs() + + logger.foreach(_.downloadingArtifact(artifact.url)) + + val url = new URL(artifact.url) + val b = Array.fill[Byte](Files.bufferSize)(0) + val in = new BufferedInputStream(url.openStream(), Files.bufferSize) + try { - @tailrec - def helper(): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - helper() + val out = new FileOutputStream(file) + try { + @tailrec + def helper(): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + helper() + } } - } - helper() - } finally out.close() - } finally in.close() + helper() + } finally out.close() + } finally in.close() - logger.foreach(_.downloadedArtifact(artifact.url, success = true)) - \/-(file) - } - catch { case e: Exception => - logger.foreach(_.downloadedArtifact(artifact.url, success = false)) - -\/(e.getMessage) + logger.foreach(_.downloadedArtifact(artifact.url, success = true)) + \/-(file) + } + catch { case e: Exception => + logger.foreach(_.downloadedArtifact(artifact.url, success = false)) + -\/(e.getMessage) + } } } - } - EitherT(cachePolicy(locally)(remote)) + EitherT(cachePolicy(locally)(remote)) + } } } From 6a91ddfacad902e0fe5c191e293e0a5dd1e6887a Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 00:59:07 +0100 Subject: [PATCH 02/11] ResolutionProcess API --- .../main/scala/coursier/cli/Coursier.scala | 79 ++++++++----- .../main/scala/coursier/core/package.scala | 2 +- .../main/scala/coursier/core/Resolution.scala | 24 ++-- .../coursier/core/ResolutionProcess.scala | 88 +++++++++++++++ core/src/main/scala/coursier/package.scala | 14 ++- .../scala/coursier/test/CentralTests.scala | 14 ++- .../scala/coursier/test/ResolutionTests.scala | 105 ++++++++---------- .../test/scala/coursier/test/package.scala | 2 +- web/src/main/scala/coursier/web/Backend.scala | 3 +- 9 files changed, 224 insertions(+), 107 deletions(-) create mode 100644 core/src/main/scala/coursier/core/ResolutionProcess.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index b1cbef11b..da091496d 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -12,8 +12,11 @@ import scalaz.concurrent.Task case class Coursier(scope: List[String], keepOptional: Boolean, fetch: Boolean, + verbose: List[Unit], @ExtraName("N") maxIterations: Int = 100) extends App { + val verbose0 = verbose.length + val scopes0 = if (scope.isEmpty) List(Scope.Compile, Scope.Runtime) else scope.map(Parse.scope) @@ -24,44 +27,48 @@ case class Coursier(scope: List[String], def fileRepr(f: File) = f.toString - val logger: MetadataFetchLogger with FilesLogger = new MetadataFetchLogger with FilesLogger { - def println(s: String) = Console.err.println(s) + val logger: Option[MetadataFetchLogger with FilesLogger] = + if (verbose0 <= 1) None + else Some( + new MetadataFetchLogger with FilesLogger { + def println(s: String) = Console.err.println(s) - def downloading(url: String) = - println(s"Downloading $url") - def downloaded(url: String, success: Boolean) = - println( - if (success) s"Downloaded $url" - else s"Failed to download $url" - ) - def readingFromCache(f: File) = { - println(s"Reading ${fileRepr(f)} from cache") - } - def puttingInCache(f: File) = - println(s"Writing ${fileRepr(f)} in cache") + def downloading(url: String) = + println(s"Downloading $url") + def downloaded(url: String, success: Boolean) = + println( + if (success) s"Downloaded $url" + else s"Failed to download $url" + ) + def readingFromCache(f: File) = { + println(s"Reading ${fileRepr(f)} from cache") + } + def puttingInCache(f: File) = + println(s"Writing ${fileRepr(f)} in cache") - def foundLocally(f: File) = - println(s"Found locally ${fileRepr(f)}") - def downloadingArtifact(url: String) = - println(s"Downloading $url") - def downloadedArtifact(url: String, success: Boolean) = - println( - if (success) s"Downloaded $url" - else s"Failed to download $url" - ) - } + def foundLocally(f: File) = + println(s"Found locally ${fileRepr(f)}") + def downloadingArtifact(url: String) = + println(s"Downloading $url") + def downloadedArtifact(url: String, success: Boolean) = + println( + if (success) s"Downloaded $url" + else s"Failed to download $url" + ) + } + ) val cachedMavenCentral = repository.mavenCentral.copy( fetchMetadata = repository.mavenCentral.fetchMetadata.copy( cache = Some(centralCacheDir), - logger = Some(logger) + logger = logger ) ) val repositories = Seq[Repository]( cachedMavenCentral, repository.ivy2Local.copy( fetchMetadata = repository.ivy2Local.fetchMetadata.copy( - logger = Some(logger) + logger = logger ) ) ) @@ -71,7 +78,7 @@ case class Coursier(scope: List[String], .partition(_.length == 3) if (splitDependencies.isEmpty) { - Console.err.println("Usage: coursier dependencies...") + CaseApp.printUsage[Coursier]() sys exit 1 } @@ -94,7 +101,21 @@ case class Coursier(scope: List[String], filter = Some(dep => (keepOptional || !dep.optional) && scopes(dep.scope)) ) - val res = startRes.last(fetchFrom(repositories), maxIterations).run + val fetchQuiet = fetchSeveralFrom(repositories) + val fetch0 = + if (verbose == 0) fetchQuiet + else { + modVers: Seq[(Module, String)] => + val print = Task{ + println(s"Getting ${modVers.length} project definition(s)") + } + + print.flatMap(_ => fetchQuiet(modVers)) + } + + val res = ResolutionProcess(startRes) + .run(fetch0, maxIterations) + .run if (!res.isDone) { Console.err.println(s"Maximum number of iteration reached!") @@ -140,7 +161,7 @@ case class Coursier(scope: List[String], cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir ), () => ???, - Some(logger) + logger ) val tasks = artifacts.map(files.file(_, cachePolicy).run) diff --git a/core-jvm/src/main/scala/coursier/core/package.scala b/core-jvm/src/main/scala/coursier/core/package.scala index 3d7851006..0f0a5e5b8 100644 --- a/core-jvm/src/main/scala/coursier/core/package.scala +++ b/core-jvm/src/main/scala/coursier/core/package.scala @@ -6,7 +6,7 @@ import scalaz.concurrent.Task package object core { def resolution(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], + fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], filter: Option[Dependency => Boolean], profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]): Stream[Resolution] = { diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index 1f771a258..1e844746f 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -24,10 +24,10 @@ object Resolution { */ def find(repositories: Seq[Repository], module: Module, - version: String): EitherT[Task, List[String], (Repository, Project)] = { + version: String): EitherT[Task, Seq[String], (Repository, Project)] = { val lookups = repositories.map(repo => repo -> repo.find(module, version).run) - val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[List[String] \/ (Repository, Project)]) { + val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Repository, Project)]) { case (acc, (repo, t)) => acc.flatMap { case -\/(errors) => @@ -36,7 +36,7 @@ object Resolution { if (project.module == module) \/-((repo, project)) else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") ) - .leftMap(error => error :: errors) + .leftMap(error => error +: errors) ) case res @ \/-(_) => @@ -456,16 +456,20 @@ case class Resolution(rootDependencies: Set[Dependency], * If no module info is missing, the next state of the resolution, which can be immediately calculated. * Else, the current resolution itself. */ - def nextIfNoMissing: Resolution = { + @tailrec + final def nextIfNoMissing: Resolution = { val missing = missingFromCache - if (missing.isEmpty) nextNoMissingUnsafe - else this + if (missing.isEmpty) { + val next0 = nextNoMissingUnsafe + if (next0 == this) this + else next0.nextIfNoMissing + } else this } /** * Do a new iteration, fetching the missing modules along the way. */ - def next(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { + def next(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)]): Task[Resolution] = { val missing = missingFromCache if (missing.isEmpty) Task.now(nextNoMissingUnsafe) else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing) @@ -575,7 +579,7 @@ case class Resolution(rootDependencies: Set[Dependency], * Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache. */ def fetch(modules: Seq[ModuleVersion], - fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)]): Task[Resolution] = { + fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)]): Task[Resolution] = { val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) @@ -598,7 +602,7 @@ case class Resolution(rootDependencies: Set[Dependency], } } - def last(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], maxIterations: Int = -1): Task[Resolution] = { + def last(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], maxIterations: Int = -1): Task[Resolution] = { if (maxIterations == 0 || isDone) Task.now(this) else { next(fetchModule) @@ -606,7 +610,7 @@ case class Resolution(rootDependencies: Set[Dependency], } } - def stream(fetchModule: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], run: Task[Resolution] => Resolution): Stream[Resolution] = { + def stream(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], run: Task[Resolution] => Resolution): Stream[Resolution] = { this #:: { if (isDone) Stream.empty else run(next(fetchModule)).stream(fetchModule, run) diff --git a/core/src/main/scala/coursier/core/ResolutionProcess.scala b/core/src/main/scala/coursier/core/ResolutionProcess.scala new file mode 100644 index 000000000..bcb49274c --- /dev/null +++ b/core/src/main/scala/coursier/core/ResolutionProcess.scala @@ -0,0 +1,88 @@ +package coursier +package core + +import scalaz._ +import scala.annotation.tailrec + + +sealed trait ResolutionProcess { + def run[F[_]](fetch: ResolutionProcess.Fetch[F], + maxIterations: Int = -1) + (implicit F: Monad[F]): F[Resolution] = { + + if (maxIterations == 0) F.point(current) + else + this match { + case Done(res) => F.point(res) + case missing0 @ Missing(missing, _, _) => + F.bind(fetch(missing))(result => missing0.next(result).run(fetch, if (maxIterations > 0) maxIterations - 1 else maxIterations)) + case cont @ Continue(_, _) => cont.nextNoCont.run(fetch) + } + } + + def current: Resolution +} + +case class Missing(missing: Seq[(Module, String)], + current: Resolution, + cont: Resolution => ResolutionProcess) extends ResolutionProcess { + + def next(results: ResolutionProcess.FetchResult): ResolutionProcess = { + + val errors = results.collect{case (modVer, -\/(errs)) => modVer -> errs } + val successes = results.collect{case (modVer, \/-(repoProj)) => modVer -> repoProj } + + val depMgmtMissing0 = successes + .map{case (_, (_, proj)) => current.dependencyManagementMissing(proj) } + .fold(Set.empty)(_ ++ _) + + val depMgmtMissing = depMgmtMissing0 -- results.map(_._1) + + def cont0(res: Resolution) = { + val res0 = + successes.foldLeft(res){case (acc, (modVer, (repo, proj))) => + acc.copy(projectCache = acc.projectCache + ( + modVer -> (repo, acc.withDependencyManagement(proj)) + )) + } + Continue(res0, cont) + } + + val current0 = current + .copy(errorCache = current.errorCache ++ errors) + + if (depMgmtMissing.isEmpty) cont0(current0) + else Missing(depMgmtMissing.toSeq, current0, cont0) + } + +} + +case class Continue(current: Resolution, + cont: Resolution => ResolutionProcess) extends ResolutionProcess { + + def next: ResolutionProcess = cont(current) + + @tailrec final def nextNoCont: ResolutionProcess = + next match { + case nextCont: Continue => nextCont.nextNoCont + case other => other + } + +} + +case class Done(resolution: Resolution) extends ResolutionProcess { + def current: Resolution = resolution +} + +object ResolutionProcess { + def apply(resolution: Resolution): ResolutionProcess = { + val resolution0 = resolution.nextIfNoMissing + + if (resolution0.isDone) Done(resolution0) + else Missing(resolution0.missingFromCache.toSeq, resolution0, apply) + } + + type FetchResult = Seq[((Module, String), Seq[String] \/ (Repository, Project))] + type Fetch[F[_]] = Seq[(Module, String)] => F[FetchResult] +} + diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 2cfd39143..7c3f35215 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -1,4 +1,4 @@ -import scalaz.EitherT +import scalaz.{ EitherT, \/ } import scalaz.concurrent.Task /** @@ -67,9 +67,15 @@ package object coursier { type Repository = core.Repository - def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, List[String], (Repository, Project)] = + def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)] = modVersion => core.Resolution.find(repositories, modVersion._1, modVersion._2) + def fetchSeveralFrom(repositories: Seq[Repository]): Seq[ModuleVersion] => Task[Seq[(ModuleVersion, Seq[String] \/ (Repository, Project))]] = { + val fetchOne = fetchFrom(repositories) + modVers => + Task.gatherUnordered(modVers.map(modVer => fetchOne(modVer).run.map(modVer -> _))) + } + type Resolution = core.Resolution object Resolution { val empty = apply() @@ -84,7 +90,7 @@ package object coursier { } def resolve(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], + fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], maxIterations: Option[Int] = Some(200), filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { @@ -110,4 +116,6 @@ package object coursier { type MavenRepository[G <: core.FetchMetadata] = core.MavenRepository[G] val MavenRepository: core.MavenRepository.type = core.MavenRepository + type ResolutionProcess = core.ResolutionProcess + val ResolutionProcess: core.ResolutionProcess.type = core.ResolutionProcess } diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index f8348043f..4a1a90a26 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -12,6 +12,12 @@ object CentralTests extends TestSuite { repository.mavenCentral ) + def resolve(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { + ResolutionProcess(Resolution(deps, filter = filter)) + .run(fetchSeveralFrom(repositories)) + .runF + } + def repr(dep: Dependency) = s"${dep.module.organization}:${dep.module.name}:${dep.attributes.`type`}:${Some(dep.attributes.classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" @@ -20,7 +26,7 @@ object CentralTests extends TestSuite { val expected = await(textResource(s"resolutions/${module.organization}:${module.name}:$version")).split('\n').toSeq val dep = Dependency(module, version) - val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) + val res = await(resolve(Set(dep))) val result = res.dependencies.toVector.map(repr).sorted.distinct @@ -34,7 +40,7 @@ object CentralTests extends TestSuite { 'logback{ async { val dep = Dependency(Module("ch.qos.logback", "logback-classic"), "1.1.3") - val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) + val res = await(resolve(Set(dep))) .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( @@ -50,7 +56,7 @@ object CentralTests extends TestSuite { 'asm{ async { val dep = Dependency(Module("org.ow2.asm", "asm-commons"), "5.0.2") - val res = await(resolve(Set(dep), fetchFrom(repositories)).runF) + val res = await(resolve(Set(dep))) .copy(projectCache = Map.empty, errorCache = Map.empty) // No validating these here val expected = Resolution( @@ -66,7 +72,7 @@ object CentralTests extends TestSuite { 'jodaVersionInterval{ async { val dep = Dependency(Module("joda-time", "joda-time"), "[2.2,2.8]") - val res0 = await(resolve(Set(dep), fetchFrom(repositories)).runF) + val res0 = await(resolve(Set(dep))) val res = res0.copy(projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( diff --git a/core/src/test/scala/coursier/test/ResolutionTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala index 4734bd0f5..6902b4e86 100644 --- a/core/src/test/scala/coursier/test/ResolutionTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -8,6 +8,12 @@ import coursier.test.compatibility._ object ResolutionTests extends TestSuite { + def resolve0(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { + ResolutionProcess(Resolution(deps, filter = filter)) + .run(fetchSeveralFrom(repositories)) + .runF + } + implicit class ProjectOps(val p: Project) extends AnyVal { def kv: (ModuleVersion, (Repository, Project)) = p.moduleVersion -> (testRepository, p) } @@ -149,10 +155,9 @@ object ResolutionTests extends TestSuite { val tests = TestSuite { 'empty{ async{ - val res = await(resolve( - Set.empty, - fetchFrom(repositories) - ).runF) + val res = await(resolve0( + Set.empty + )) assert(res == Resolution.empty) } @@ -160,10 +165,9 @@ object ResolutionTests extends TestSuite { 'notFound{ async { val dep = Dependency(Module("acme", "playy"), "2.4.0") - val res = await(resolve( - Set(dep), - fetchFrom(repositories) - ).runF) + val res = await(resolve0( + Set(dep) + )) val expected = Resolution( rootDependencies = Set(dep), @@ -177,10 +181,9 @@ object ResolutionTests extends TestSuite { 'single{ async { val dep = Dependency(Module("acme", "config"), "1.3.0") - val res = await(resolve( - Set(dep), - fetchFrom(repositories) - ).runF) + val res = await(resolve0( + Set(dep) + )) val expected = Resolution( rootDependencies = Set(dep), @@ -195,10 +198,9 @@ object ResolutionTests extends TestSuite { async { val dep = Dependency(Module("acme", "play"), "2.4.0") val trDep = Dependency(Module("acme", "play-json"), "2.4.0") - val res = await(resolve( - Set(dep), - fetchFrom(repositories) - ).runF) + val res = await(resolve0( + Set(dep) + )) val expected = Resolution( rootDependencies = Set(dep), @@ -219,10 +221,9 @@ object ResolutionTests extends TestSuite { Dependency(Module("acme", "play-json"), "2.4.0"), Dependency(Module("acme", "config"), "1.3.0") ) - val res = await(resolve( - Set(dep), - fetchFrom(repositories) - ).runF) + val res = await(resolve0( + Set(dep) + )) val expected = Resolution( rootDependencies = Set(dep), @@ -244,10 +245,9 @@ object ResolutionTests extends TestSuite { Dependency(Module("acme", "play-json"), "2.4.0", exclusions = Set(("acme", "config"))) ) - val res = await(resolve( - Set(dep), - fetchFrom(repositories) - ).runF) + val res = await(resolve0( + Set(dep) + )) val expected = Resolution( rootDependencies = Set(dep), @@ -269,10 +269,9 @@ object ResolutionTests extends TestSuite { Dependency(Module("acme", "play-json"), "2.4.0", exclusions = Set(("*", "config"))) ) - val res = await(resolve( - Set(dep), - fetchFrom(repositories) - ).runF) + val res = await(resolve0( + Set(dep) + )) val expected = Resolution( rootDependencies = Set(dep), @@ -288,11 +287,10 @@ object ResolutionTests extends TestSuite { 'filter{ async { val dep = Dependency(Module("hudsucker", "mail"), "10.0") - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None) + )).copy(filter = None) val expected = Resolution( rootDependencies = Set(dep), @@ -312,11 +310,10 @@ object ResolutionTests extends TestSuite { Dependency(Module("acme", "play"), "2.4.0", exclusions = Set(("acme", "play-json"))) ) - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -332,11 +329,10 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("org.gnu", "glib"), "13.4"), Dependency(Module("org.gnome", "desktop"), "7.0")) - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -351,11 +347,10 @@ object ResolutionTests extends TestSuite { val dep = Dependency(Module("com.mailapp", "mail-client"), "2.1") val trDeps = Seq( Dependency(Module("gov.nsa", "secure-pgp"), "10.0", exclusions = Set(("*", "crypto")))) - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -368,11 +363,10 @@ object ResolutionTests extends TestSuite { 'depMgmtInParentDeps{ async { val dep = Dependency(Module("com.thoughtworks.paranamer", "paranamer"), "2.6") - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -387,11 +381,10 @@ object ResolutionTests extends TestSuite { val dep = Dependency(Module("com.github.dummy", "libb"), "0.3.3") val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -408,11 +401,10 @@ object ResolutionTests extends TestSuite { val dep = Dependency(Module("com.github.dummy", "libb"), version) val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -431,11 +423,10 @@ object ResolutionTests extends TestSuite { val dep = Dependency(Module("com.github.dummy", "libb"), "0.4.2") val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -453,11 +444,10 @@ object ResolutionTests extends TestSuite { Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))), Dependency(Module("an-org", "another-lib"), "1.0", optional = true), Dependency(Module("an-org", "a-name"), "1.0", optional = true)) - val res = await(resolve( + val res = await(resolve0( Set(dep), - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), @@ -477,11 +467,10 @@ object ResolutionTests extends TestSuite { Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))), Dependency(Module("an-org", "another-lib"), "1.0", optional = true), Dependency(Module("an-org", "a-name"), "1.0", optional = true)) - val res = await(resolve( + val res = await(resolve0( deps, - fetchFrom(repositories), filter = Some(_.scope == Scope.Compile) - ).runF).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) + )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( rootDependencies = deps, diff --git a/core/src/test/scala/coursier/test/package.scala b/core/src/test/scala/coursier/test/package.scala index af3c811e8..cf746c90e 100644 --- a/core/src/test/scala/coursier/test/package.scala +++ b/core/src/test/scala/coursier/test/package.scala @@ -10,7 +10,7 @@ package object test { } def resolve(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, List[String], (Repository, Project)], + fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], maxIterations: Option[Int] = Some(200), filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 528e4cd04..c6f15dc7c 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -127,7 +127,8 @@ class Backend($: BackendScope[Unit, State]) { filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) ) - res.last(fetchFrom(s.repositories.map(r => r.copy(fetchMetadata = r.fetchMetadata.copy(logger = Some(logger))))), 100) + ResolutionProcess(res) + .run(fetchSeveralFrom(s.repositories.map(r => r.copy(fetchMetadata = r.fetchMetadata.copy(logger = Some(logger))))), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context From b010d4b620a9fed268b5e5f477fef97817f51562 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 00:59:08 +0100 Subject: [PATCH 03/11] Rework Resolution iteration and artifacts API --- .../main/scala/coursier/cli/Coursier.scala | 78 +++-- ...DefaultFetchMetadata.scala => Fetch.scala} | 22 +- .../test/scala/coursier/test/JsTests.scala | 10 +- ...DefaultFetchMetadata.scala => Fetch.scala} | 28 +- .../main/scala/coursier/core/package.scala | 31 -- .../coursier/test/compatibility/package.scala | 4 +- .../scala/coursier/core/Definitions.scala | 5 + .../main/scala/coursier/core/Repository.scala | 283 +++++++++++------- .../main/scala/coursier/core/Resolution.scala | 99 +----- .../coursier/core/ResolutionProcess.scala | 6 +- core/src/main/scala/coursier/package.scala | 36 +-- .../scala/coursier/repository/package.scala | 14 - .../scala/coursier/test/CentralTests.scala | 7 +- .../scala/coursier/test/ResolutionTests.scala | 9 +- .../scala/coursier/test/TestRepository.scala | 12 +- .../test/scala/coursier/test/package.scala | 17 -- files/src/main/scala/coursier/Files.scala | 2 +- web/src/main/scala/coursier/web/Backend.scala | 38 ++- 18 files changed, 308 insertions(+), 393 deletions(-) rename core-js/src/main/scala/coursier/core/{DefaultFetchMetadata.scala => Fetch.scala} (83%) rename core-jvm/src/main/scala/coursier/core/{DefaultFetchMetadata.scala => Fetch.scala} (78%) delete mode 100644 core-jvm/src/main/scala/coursier/core/package.scala delete mode 100644 core/src/main/scala/coursier/repository/package.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index da091496d..6f2d2e34c 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -4,15 +4,14 @@ package cli import java.io.File import caseapp._ -import coursier.core.{CachePolicy, Parse} -import coursier.core.MetadataFetchLogger +import coursier.core.{Fetch, Parse, Repository}, Repository.CachePolicy import scalaz.concurrent.Task case class Coursier(scope: List[String], keepOptional: Boolean, fetch: Boolean, - verbose: List[Unit], + @ExtraName("v") verbose: List[Unit], @ExtraName("N") maxIterations: Int = 100) extends App { val verbose0 = verbose.length @@ -27,48 +26,45 @@ case class Coursier(scope: List[String], def fileRepr(f: File) = f.toString - val logger: Option[MetadataFetchLogger with FilesLogger] = - if (verbose0 <= 1) None - else Some( - new MetadataFetchLogger with FilesLogger { - def println(s: String) = Console.err.println(s) + val logger: Fetch.Logger with FilesLogger = + new Fetch.Logger with FilesLogger { + def println(s: String) = Console.err.println(s) - def downloading(url: String) = - println(s"Downloading $url") - def downloaded(url: String, success: Boolean) = - println( - if (success) s"Downloaded $url" - else s"Failed to download $url" - ) - def readingFromCache(f: File) = { - println(s"Reading ${fileRepr(f)} from cache") - } - def puttingInCache(f: File) = - println(s"Writing ${fileRepr(f)} in cache") - - def foundLocally(f: File) = - println(s"Found locally ${fileRepr(f)}") - def downloadingArtifact(url: String) = - println(s"Downloading $url") - def downloadedArtifact(url: String, success: Boolean) = - println( - if (success) s"Downloaded $url" - else s"Failed to download $url" - ) + def downloading(url: String) = + println(s"Downloading $url") + def downloaded(url: String, success: Boolean) = + println( + if (success) s"Downloaded $url" + else s"Failed to download $url" + ) + def readingFromCache(f: File) = { + println(s"Reading ${fileRepr(f)} from cache") } - ) + def puttingInCache(f: File) = + println(s"Writing ${fileRepr(f)} in cache") - val cachedMavenCentral = repository.mavenCentral.copy( - fetchMetadata = repository.mavenCentral.fetchMetadata.copy( + def foundLocally(f: File) = + println(s"Found locally ${fileRepr(f)}") + def downloadingArtifact(url: String) = + println(s"Downloading $url") + def downloadedArtifact(url: String, success: Boolean) = + println( + if (success) s"Downloaded $url" + else s"Failed to download $url" + ) + } + + val cachedMavenCentral = Repository.mavenCentral.copy( + fetch = Repository.mavenCentral.fetch.copy( cache = Some(centralCacheDir), - logger = logger + logger = if (verbose0 <= 1) None else Some(logger) ) ) val repositories = Seq[Repository]( cachedMavenCentral, - repository.ivy2Local.copy( - fetchMetadata = repository.ivy2Local.fetchMetadata.copy( - logger = logger + Repository.ivy2Local.copy( + fetch = Repository.ivy2Local.fetch.copy( + logger = if (verbose0 <= 1) None else Some(logger) ) ) ) @@ -101,9 +97,9 @@ case class Coursier(scope: List[String], filter = Some(dep => (keepOptional || !dep.optional) && scopes(dep.scope)) ) - val fetchQuiet = fetchSeveralFrom(repositories) + val fetchQuiet = Repository.fetchSeveralFrom(repositories) val fetch0 = - if (verbose == 0) fetchQuiet + if (verbose0 == 0) fetchQuiet else { modVers: Seq[(Module, String)] => val print = Task{ @@ -158,10 +154,10 @@ case class Coursier(scope: List[String], val files = new Files( Seq( - cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir + cachedMavenCentral.fetch.root -> centralFilesCacheDir ), () => ???, - logger + if (verbose0 <= 0) None else Some(logger) ) val tasks = artifacts.map(files.file(_, cachePolicy).run) diff --git a/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-js/src/main/scala/coursier/core/Fetch.scala similarity index 83% rename from core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala rename to core-js/src/main/scala/coursier/core/Fetch.scala index d41b995bf..682b58f4b 100644 --- a/core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala +++ b/core-js/src/main/scala/coursier/core/Fetch.scala @@ -12,7 +12,7 @@ import js.Dynamic.{global => g} import scala.scalajs.js.timers._ -object DefaultFetchMetadata { +object Fetch { def encodeURIComponent(s: String): String = g.encodeURIComponent(s).asInstanceOf[String] @@ -70,25 +70,27 @@ object DefaultFetchMetadata { p.future } + trait Logger { + def fetching(url: String): Unit + def fetched(url: String): Unit + def other(url: String, msg: String): Unit + } + } -trait Logger { - def fetching(url: String): Unit - def fetched(url: String): Unit - def other(url: String, msg: String): Unit -} +case class Fetch(root: String, + logger: Option[Fetch.Logger] = None) { -case class DefaultFetchMetadata(root: String, - logger: Option[Logger] = None) extends FetchMetadata { def apply(artifact: Artifact, - cachePolicy: CachePolicy): EitherT[Task, String, String] = { + cachePolicy: Repository.CachePolicy): EitherT[Task, String, String] = { EitherT( Task { implicit ec => - DefaultFetchMetadata.get(root + artifact.url) + Fetch.get(root + artifact.url) .map(\/-(_)) .recover{case e: Exception => -\/(e.getMessage)} } ) } + } \ No newline at end of file diff --git a/core-js/src/test/scala/coursier/test/JsTests.scala b/core-js/src/test/scala/coursier/test/JsTests.scala index 9738b2600..82533d4e8 100644 --- a/core-js/src/test/scala/coursier/test/JsTests.scala +++ b/core-js/src/test/scala/coursier/test/JsTests.scala @@ -1,7 +1,7 @@ package coursier package test -import coursier.core.DefaultFetchMetadata +import coursier.core.{Fetch, Repository} import coursier.test.compatibility._ import utest._ @@ -11,14 +11,14 @@ import scala.concurrent.{Future, Promise} object JsTests extends TestSuite { val tests = TestSuite { - 'promise { + 'promise{ val p = Promise[Unit]() Future(p.success(())) p.future } 'get{ - DefaultFetchMetadata.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + Fetch.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") .map(core.compatibility.xmlParse) .map{ xml => assert(xml.right.toOption.exists(_.label == "project")) @@ -26,9 +26,9 @@ object JsTests extends TestSuite { } 'getProj{ - repository.mavenCentral + Repository.mavenCentral .find(Module("ch.qos.logback", "logback-classic"), "1.1.3") - .map{ proj => + .map{case (_, proj) => assert(proj.parent == Some(Module("ch.qos.logback", "logback-parent"), "1.1.3")) } .run diff --git a/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala b/core-jvm/src/main/scala/coursier/core/Fetch.scala similarity index 78% rename from core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala rename to core-jvm/src/main/scala/coursier/core/Fetch.scala index 70bd9dea9..73d6b509f 100644 --- a/core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala +++ b/core-jvm/src/main/scala/coursier/core/Fetch.scala @@ -8,20 +8,15 @@ import scala.io.Codec import scalaz._, Scalaz._ import scalaz.concurrent.Task -trait MetadataFetchLogger { - def downloading(url: String): Unit - def downloaded(url: String, success: Boolean): Unit - def readingFromCache(f: File): Unit - def puttingInCache(f: File): Unit -} - -case class DefaultFetchMetadata(root: String, - cache: Option[File] = None, - logger: Option[MetadataFetchLogger] = None) extends FetchMetadata { +case class Fetch(root: String, + cache: Option[File] = None, + logger: Option[Fetch.Logger] = None) { val isLocal = root.startsWith("file:///") - def apply(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, String] = { + def apply(artifact: Artifact, + cachePolicy: Repository.CachePolicy): EitherT[Task, String, String] = { + def locally(eitherFile: String \/ File) = { Task { for { @@ -49,7 +44,7 @@ case class DefaultFetchMetadata(root: String, val url = new URL(urlStr) def log = Task(logger.foreach(_.downloading(urlStr))) - def get = DefaultFetchMetadata.readFully(url.openStream()) + def get = Fetch.readFully(url.openStream()) log.flatMap(_ => get) } @@ -75,7 +70,14 @@ case class DefaultFetchMetadata(root: String, } -object DefaultFetchMetadata { +object Fetch { + + trait Logger { + def downloading(url: String): Unit + def downloaded(url: String, success: Boolean): Unit + def readingFromCache(f: File): Unit + def puttingInCache(f: File): Unit + } def readFullySync(is: InputStream) = { val buffer = new ByteArrayOutputStream() diff --git a/core-jvm/src/main/scala/coursier/core/package.scala b/core-jvm/src/main/scala/coursier/core/package.scala deleted file mode 100644 index 0f0a5e5b8..000000000 --- a/core-jvm/src/main/scala/coursier/core/package.scala +++ /dev/null @@ -1,31 +0,0 @@ -package coursier - -import scalaz.EitherT -import scalaz.concurrent.Task - -package object core { - - def resolution(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], - filter: Option[Dependency => Boolean], - profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]): Stream[Resolution] = { - - val startResolution = Resolution( - dependencies, Set.empty, Set.empty, - Map.empty, Map.empty, - filter, - profileActivation - ) - - def helper(resolution: Resolution): Stream[Resolution] = { - if (resolution.isDone) Stream() - else { - val nextRes = resolution.next(fetch).run - nextRes #:: helper(nextRes) - } - } - - startResolution #:: helper(startResolution) - } - -} diff --git a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala index 10c93544c..563d745ac 100644 --- a/core-jvm/src/test/scala/coursier/test/compatibility/package.scala +++ b/core-jvm/src/test/scala/coursier/test/compatibility/package.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.core.DefaultFetchMetadata +import coursier.core.Fetch import scala.concurrent.{ExecutionContext, Future} import scalaz.concurrent.Task @@ -17,7 +17,7 @@ package object compatibility { def is = getClass.getClassLoader .getResource(path).openStream() - new String(DefaultFetchMetadata.readFullySync(is), "UTF-8") + new String(Fetch.readFullySync(is), "UTF-8") } } diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index 75c775633..29f46793f 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -102,4 +102,9 @@ object Artifact { val javadocSig = "javadoc-pgp" val javadocSigMd5 = "md5-javadoc-pgp" val javadocSigSha1 = "sha1-javadoc-pgp" + + trait Source { + def artifacts(dependency: Dependency, + project: Project): Seq[Artifact] + } } diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index a8bb44f7d..088b96961 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -1,45 +1,113 @@ package coursier.core +import coursier.core.Resolution.ModuleVersion + import scalaz.{-\/, \/-, \/, EitherT} import scalaz.concurrent.Task import coursier.core.compatibility.encodeURIComponent trait Repository { - def find(module: Module, version: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Project] - def artifacts(dependency: Dependency, project: Project): Seq[Artifact] -} - -sealed trait CachePolicy { - def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T] - - def saving[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T])(save: => T => Task[Unit]): Task[E \/ T] = - apply(local)(CachePolicy.saving(remote)(save)) -} - -object CachePolicy { - def saving[E,T](remote: => Task[E \/ T])(save: T => Task[Unit]): Task[E \/ T] = { - for { - res <- remote - _ <- res.fold(_ => Task.now(()), t => save(t)) - } yield res - } - - case object Default extends CachePolicy { - def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T] = - local.flatMap(res => if (res.isLeft) remote else Task.now(res)) - } - case object LocalOnly extends CachePolicy { - def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T] = - local - } - case object ForceDownload extends CachePolicy { - def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T] = - remote - } + def find(module: Module, + version: String, + cachePolicy: Repository.CachePolicy = Repository.CachePolicy.Default): EitherT[Task, String, (Artifact.Source, Project)] } object Repository { + + val mavenCentral = MavenRepository(Fetch("https://repo1.maven.org/maven2/")) + + val sonatypeReleases = MavenRepository(Fetch("https://oss.sonatype.org/content/repositories/releases/")) + val sonatypeSnapshots = MavenRepository(Fetch("https://oss.sonatype.org/content/repositories/snapshots/")) + + lazy val ivy2Local = MavenRepository(Fetch("file://" + sys.props("user.home") + "/.ivy2/local/"), ivyLike = true) + + + /** + * Try to find `module` among `repositories`. + * + * Look at `repositories` from the left, one-by-one, and stop at first success. + * Else, return all errors, in the same order. + * + * The `version` field of the returned `Project` in case of success may not be + * equal to the provided one, in case the latter is not a specific + * version (e.g. version interval). Which version get chosen depends on + * the repository implementation. + */ + def find(repositories: Seq[Repository], + module: Module, + version: String): EitherT[Task, Seq[String], (Artifact.Source, Project)] = { + + val lookups = repositories.map(repo => repo -> repo.find(module, version).run) + val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Artifact.Source, Project)]) { + case (acc, (repo, t)) => + acc.flatMap { + case -\/(errors) => + t.map(res => res + .flatMap{case (source, project) => + if (project.module == module) \/-((source, project)) + else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") + } + .leftMap(error => error +: errors) + ) + + case res @ \/-(_) => + Task.now(res) + } + } + + EitherT(task.map(_.leftMap(_.reverse))).map {case x @ (_, proj) => + assert(proj.module == module) + x + } + } + + def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, Seq[String], (Artifact.Source, Project)] = + modVersion => find(repositories, modVersion._1, modVersion._2) + + def fetchSeveralFrom(repositories: Seq[Repository]): Seq[ModuleVersion] => Task[Seq[(ModuleVersion, Seq[String] \/ (Artifact.Source, Project))]] = { + val fetchOne = fetchFrom(repositories) + modVers => + Task.gatherUnordered(modVers.map(modVer => fetchOne(modVer).run.map(modVer -> _))) + } + + sealed trait CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] + + def saving[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]) + (save: => T => Task[Unit]): Task[E \/ T] = + apply(local)(CachePolicy.saving(remote)(save)) + } + + object CachePolicy { + def saving[E,T](remote: => Task[E \/ T]) + (save: T => Task[Unit]): Task[E \/ T] = { + for { + res <- remote + _ <- res.fold(_ => Task.now(()), t => save(t)) + } yield res + } + + case object Default extends CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] = + local + .flatMap(res => if (res.isLeft) remote else Task.now(res)) + } + case object LocalOnly extends CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] = + local + } + case object ForceDownload extends CachePolicy { + def apply[E,T](local: => Task[E \/ T]) + (remote: => Task[E \/ T]): Task[E \/ T] = + remote + } + } + implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { def withDefaultChecksums: Artifact = underlying.copy(extra = underlying.extra ++ Seq( @@ -72,24 +140,88 @@ object Repository { } } -trait FetchMetadata { - def root: String - def apply(artifact: Artifact, - cachePolicy: CachePolicy): EitherT[Task, String, String] +object MavenRepository { + + def ivyLikePath(org: String, + name: String, + version: String, + subDir: String, + baseSuffix: String, + ext: String) = + Seq( + org, + name, + version, + subDir, + s"$name$baseSuffix.$ext" + ) + + case class Source(root: String, ivyLike: Boolean) extends Artifact.Source { + import Repository._ + + def artifacts(dependency: Dependency, + project: Project): Seq[Artifact] = { + + def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = + MavenRepository.ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext) + + val path = + if (ivyLike) + ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) + else + dependency.module.organization.split('.').toSeq ++ Seq( + dependency.module.name, + project.version, + s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + ) + + var artifact = + Artifact( + root + path.mkString("/"), + Map.empty, + dependency.attributes + ) + .withDefaultChecksums + + if (dependency.attributes.`type` == "jar") { + artifact = artifact.withDefaultSignature + + artifact = + if (ivyLike) { + val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") + val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") + + artifact + .copy(extra = artifact.extra ++ Map( + Artifact.sourcesMd5 -> (srcPath + ".md5"), + Artifact.sourcesSha1 -> (srcPath + ".sha1"), + Artifact.sources -> srcPath, + Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), + Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), + Artifact.sourcesSig -> (srcPath + ".asc"), + Artifact.javadocMd5 -> (javadocPath + ".md5"), + Artifact.javadocSha1 -> (javadocPath + ".sha1"), + Artifact.javadoc -> javadocPath, + Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), + Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), + Artifact.javadocSig -> (javadocPath + ".asc") + )) + } else artifact.withJavadocSources + } + + Seq(artifact) + } + } + } -case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, - ivyLike: Boolean = false) extends Repository { +case class MavenRepository(fetch: Fetch, + ivyLike: Boolean = false) extends Repository { import Repository._ + import MavenRepository._ - def ivyLikePath(org: String, name: String, version: String, subDir: String, baseSuffix: String, ext: String) = Seq( - org, - name, - version, - subDir, - s"$name$baseSuffix.$ext" - ) + val source = MavenRepository.Source(fetch.root, ivyLike) def projectArtifact(module: Module, version: String): Artifact = { val path = ( @@ -142,7 +274,7 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, versionsArtifact(module) match { case None => Task.now(-\/("Not supported")) case Some(artifact) => - fetchMetadata(artifact, cachePolicy) + fetch(artifact, cachePolicy) .run .map(eitherStr => for { @@ -161,7 +293,7 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, cachePolicy: CachePolicy): EitherT[Task, String, Project] = { EitherT { - fetchMetadata(projectArtifact(module, version), cachePolicy) + fetch(projectArtifact(module, version), cachePolicy) .run .map(eitherStr => for { @@ -176,10 +308,10 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, def find(module: Module, version: String, - cachePolicy: CachePolicy): EitherT[Task, String, Project] = { + cachePolicy: CachePolicy): EitherT[Task, String, (Artifact.Source, Project)] = { Parse.versionInterval(version).filter(_.isValid) match { - case None => findNoInterval(module, version, cachePolicy) + case None => findNoInterval(module, version, cachePolicy).map((source, _)) case Some(itv) => versions(module, cachePolicy) .flatMap { versions0 => @@ -198,65 +330,14 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F, } eitherVersion match { - case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason))) + case -\/(reason) => EitherT[Task, String, (Artifact.Source, Project)](Task.now(-\/(reason))) case \/-(version0) => findNoInterval(module, version0, cachePolicy) .map(_.copy(versions = Some(versions0))) + .map((source, _)) } } } } - def artifacts(dependency: Dependency, - project: Project): Seq[Artifact] = { - - def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = - ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext) - - val path = - if (ivyLike) - ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) - else - dependency.module.organization.split('.').toSeq ++ Seq( - dependency.module.name, - project.version, - s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" - ) - - var artifact = - Artifact( - fetchMetadata.root + path.mkString("/"), - Map.empty, - dependency.attributes - ) - .withDefaultChecksums - - if (dependency.attributes.`type` == "jar") { - artifact = artifact.withDefaultSignature - - artifact = - if (ivyLike) { - val srcPath = fetchMetadata.root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") - val javadocPath = fetchMetadata.root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") - - artifact - .copy(extra = artifact.extra ++ Map( - Artifact.sourcesMd5 -> (srcPath + ".md5"), - Artifact.sourcesSha1 -> (srcPath + ".sha1"), - Artifact.sources -> srcPath, - Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"), - Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"), - Artifact.sourcesSig -> (srcPath + ".asc"), - Artifact.javadocMd5 -> (javadocPath + ".md5"), - Artifact.javadocSha1 -> (javadocPath + ".sha1"), - Artifact.javadoc -> javadocPath, - Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"), - Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"), - Artifact.javadocSig -> (javadocPath + ".asc") - )) - } else artifact.withJavadocSources - } - - Seq(artifact) - } } \ No newline at end of file diff --git a/core/src/main/scala/coursier/core/Resolution.scala b/core/src/main/scala/coursier/core/Resolution.scala index 1e844746f..405079303 100644 --- a/core/src/main/scala/coursier/core/Resolution.scala +++ b/core/src/main/scala/coursier/core/Resolution.scala @@ -4,52 +4,12 @@ import java.util.regex.Pattern.quote import scala.annotation.tailrec import scala.collection.mutable -import scalaz.concurrent.Task -import scalaz.{EitherT, \/-, \/, -\/} +import scalaz.{\/-, -\/} object Resolution { type ModuleVersion = (Module, String) - /** - * Try to find `module` among `repositories`. - * - * Look at `repositories` from the left, one-by-one, and stop at first success. - * Else, return all errors, in the same order. - * - * The `version` field of the returned `Project` in case of success may not be - * equal to the provided one, in case the latter is not a specific - * version (e.g. version interval). Which version get chosen depends on - * the repository implementation. - */ - def find(repositories: Seq[Repository], - module: Module, - version: String): EitherT[Task, Seq[String], (Repository, Project)] = { - - val lookups = repositories.map(repo => repo -> repo.find(module, version).run) - val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Repository, Project)]) { - case (acc, (repo, t)) => - acc.flatMap { - case -\/(errors) => - t.map(res => res - .flatMap(project => - if (project.module == module) \/-((repo, project)) - else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") - ) - .leftMap(error => error +: errors) - ) - - case res @ \/-(_) => - Task.now(res) - } - } - - EitherT(task.map(_.leftMap(_.reverse))).map { case x @ (_, proj) => - assert(proj.module == module) - x - } - } - /** * Get the active profiles of `project`, using the current properties `properties`, * and `profileActivation` stating if a profile is active. @@ -328,7 +288,7 @@ object Resolution { case class Resolution(rootDependencies: Set[Dependency], dependencies: Set[Dependency], conflicts: Set[Dependency], - projectCache: Map[Resolution.ModuleVersion, (Repository, Project)], + projectCache: Map[Resolution.ModuleVersion, (Artifact.Source, Project)], errorCache: Map[Resolution.ModuleVersion, Seq[String]], filter: Option[Dependency => Boolean], profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) { @@ -466,15 +426,6 @@ case class Resolution(rootDependencies: Set[Dependency], } else this } - /** - * Do a new iteration, fetching the missing modules along the way. - */ - def next(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)]): Task[Resolution] = { - val missing = missingFromCache - if (missing.isEmpty) Task.now(nextNoMissingUnsafe) - else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing) - } - /** * Required modules for the dependency management of `project`. */ @@ -575,56 +526,14 @@ case class Resolution(rootDependencies: Set[Dependency], ) } - /** - * Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache. - */ - def fetch(modules: Seq[ModuleVersion], - fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)]): Task[Resolution] = { - - val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _)) - val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true) - gatheredLookups.flatMap{ lookupResults => - val errors0 = errorCache ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors} - val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj} - - /* - * newProjects are project definitions, fresh from the repositories. We need to add - * dependency management / inheritance-related bits to them. - */ - - newProjects.foldLeft(Task.now(copy(errorCache = errors0))) { case (accTask, (modVer, (repo, proj))) => - for { - current <- accTask - updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule) - proj0 = updated.withDependencyManagement(proj) - } yield updated.copy(projectCache = updated.projectCache + (modVer -> (repo, proj0))) - } - } - } - - def last(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], maxIterations: Int = -1): Task[Resolution] = { - if (maxIterations == 0 || isDone) Task.now(this) - else { - next(fetchModule) - .flatMap(_.last(fetchModule, if (maxIterations > 0) maxIterations - 1 else maxIterations)) - } - } - - def stream(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], run: Task[Resolution] => Resolution): Stream[Resolution] = { - this #:: { - if (isDone) Stream.empty - else run(next(fetchModule)).stream(fetchModule, run) - } - } - def minDependencies: Set[Dependency] = Orders.minDependencies(dependencies) def artifacts: Seq[Artifact] = for { dep <- minDependencies.toSeq - (repo, proj) <- projectCache.get(dep.moduleVersion).toSeq - artifact <- repo.artifacts(dep, proj) + (source, proj) <- projectCache.get(dep.moduleVersion).toSeq + artifact <- source.artifacts(dep, proj) } yield artifact def errors: Seq[(Dependency, Seq[String])] = diff --git a/core/src/main/scala/coursier/core/ResolutionProcess.scala b/core/src/main/scala/coursier/core/ResolutionProcess.scala index bcb49274c..adb4c27f8 100644 --- a/core/src/main/scala/coursier/core/ResolutionProcess.scala +++ b/core/src/main/scala/coursier/core/ResolutionProcess.scala @@ -40,9 +40,9 @@ case class Missing(missing: Seq[(Module, String)], def cont0(res: Resolution) = { val res0 = - successes.foldLeft(res){case (acc, (modVer, (repo, proj))) => + successes.foldLeft(res){case (acc, (modVer, (source, proj))) => acc.copy(projectCache = acc.projectCache + ( - modVer -> (repo, acc.withDependencyManagement(proj)) + modVer -> (source, acc.withDependencyManagement(proj)) )) } Continue(res0, cont) @@ -82,7 +82,7 @@ object ResolutionProcess { else Missing(resolution0.missingFromCache.toSeq, resolution0, apply) } - type FetchResult = Seq[((Module, String), Seq[String] \/ (Repository, Project))] + type FetchResult = Seq[((Module, String), Seq[String] \/ (Artifact.Source, Project))] type Fetch[F[_]] = Seq[(Module, String)] => F[FetchResult] } diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 7c3f35215..96ecf6c4c 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -1,5 +1,3 @@ -import scalaz.{ EitherT, \/ } -import scalaz.concurrent.Task /** * Pulls definitions from coursier.core, with default arguments. @@ -65,56 +63,28 @@ package object coursier { type Scope = core.Scope val Scope: core.Scope.type = core.Scope - type Repository = core.Repository - - def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)] = - modVersion => core.Resolution.find(repositories, modVersion._1, modVersion._2) - - def fetchSeveralFrom(repositories: Seq[Repository]): Seq[ModuleVersion] => Task[Seq[(ModuleVersion, Seq[String] \/ (Repository, Project))]] = { - val fetchOne = fetchFrom(repositories) - modVers => - Task.gatherUnordered(modVers.map(modVer => fetchOne(modVer).run.map(modVer -> _))) - } - type Resolution = core.Resolution object Resolution { val empty = apply() def apply(rootDependencies: Set[Dependency] = Set.empty, dependencies: Set[Dependency] = Set.empty, conflicts: Set[Dependency] = Set.empty, - projectCache: Map[ModuleVersion, (Repository, Project)] = Map.empty, + projectCache: Map[ModuleVersion, (Artifact.Source, Project)] = Map.empty, errorCache: Map[ModuleVersion, Seq[String]] = Map.empty, filter: Option[Dependency => Boolean] = None, profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution = core.Resolution(rootDependencies, dependencies, conflicts, projectCache, errorCache, filter, profileActivation) } - def resolve(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], - maxIterations: Option[Int] = Some(200), - filter: Option[Dependency => Boolean] = None, - profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { - - val startResolution = Resolution( - dependencies, Set.empty, Set.empty, - Map.empty, Map.empty, - filter, - profileActivation - ) - - startResolution.last(fetch, maxIterations.getOrElse(-1)) - } - type Artifact = core.Artifact object Artifact { def apply(url: String, extra: Map[String, String] = Map.empty, attributes: Attributes = Attributes()): Artifact = core.Artifact(url, extra, attributes) - } - type MavenRepository[G <: core.FetchMetadata] = core.MavenRepository[G] - val MavenRepository: core.MavenRepository.type = core.MavenRepository + type Source = core.Artifact.Source + } type ResolutionProcess = core.ResolutionProcess val ResolutionProcess: core.ResolutionProcess.type = core.ResolutionProcess diff --git a/core/src/main/scala/coursier/repository/package.scala b/core/src/main/scala/coursier/repository/package.scala deleted file mode 100644 index 93acd2c09..000000000 --- a/core/src/main/scala/coursier/repository/package.scala +++ /dev/null @@ -1,14 +0,0 @@ -package coursier - -import coursier.core.DefaultFetchMetadata - -package object repository { - - val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/")) - - val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/")) - val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/")) - - lazy val ivy2Local = MavenRepository(DefaultFetchMetadata("file://" + sys.props("user.home") + "/.ivy2/local/"), ivyLike = true) - -} diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 4a1a90a26..bf0d4ae19 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -1,6 +1,7 @@ package coursier package test +import coursier.core.Repository import utest._ import scala.async.Async.{async, await} @@ -9,12 +10,12 @@ import coursier.test.compatibility._ object CentralTests extends TestSuite { val repositories = Seq[Repository]( - repository.mavenCentral + Repository.mavenCentral ) def resolve(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { ResolutionProcess(Resolution(deps, filter = filter)) - .run(fetchSeveralFrom(repositories)) + .run(Repository.fetchSeveralFrom(repositories)) .runF } @@ -83,7 +84,7 @@ object CentralTests extends TestSuite { assert(res == expected) assert(res0.projectCache.contains(dep.moduleVersion)) - val (_, proj) = res0.projectCache(dep.moduleVersion) + val proj = res0.projectCache(dep.moduleVersion)._2 assert(proj.version == "2.8") } } diff --git a/core/src/test/scala/coursier/test/ResolutionTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala index 6902b4e86..66f7f20d1 100644 --- a/core/src/test/scala/coursier/test/ResolutionTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -1,6 +1,7 @@ package coursier package test +import coursier.core.Repository import utest._ import scala.async.Async.{async, await} @@ -10,12 +11,12 @@ object ResolutionTests extends TestSuite { def resolve0(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { ResolutionProcess(Resolution(deps, filter = filter)) - .run(fetchSeveralFrom(repositories)) + .run(Repository.fetchSeveralFrom(repositories)) .runF } implicit class ProjectOps(val p: Project) extends AnyVal { - def kv: (ModuleVersion, (Repository, Project)) = p.moduleVersion -> (testRepository, p) + def kv: (ModuleVersion, (Artifact.Source, Project)) = p.moduleVersion -> (testRepository.source, p) } val projects = Seq( @@ -146,7 +147,7 @@ object ResolutionTests extends TestSuite { ) val projectsMap = projects.map(p => p.moduleVersion -> p).toMap - val testRepository: Repository = new TestRepository(projectsMap) + val testRepository = new TestRepository(projectsMap) val repositories = Seq[Repository]( testRepository @@ -188,7 +189,7 @@ object ResolutionTests extends TestSuite { val expected = Resolution( rootDependencies = Set(dep), dependencies = Set(dep.withCompileScope), - projectCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion))) + projectCache = Map(dep.moduleVersion -> (testRepository.source, projectsMap(dep.moduleVersion))) ) assert(res == expected) diff --git a/core/src/test/scala/coursier/test/TestRepository.scala b/core/src/test/scala/coursier/test/TestRepository.scala index fcdf892ff..80afb0bb6 100644 --- a/core/src/test/scala/coursier/test/TestRepository.scala +++ b/core/src/test/scala/coursier/test/TestRepository.scala @@ -1,16 +1,18 @@ package coursier package test -import coursier.core.{Versions, CachePolicy} +import coursier.core._ -import scalaz.{-\/, \/, EitherT} +import scalaz.EitherT import scalaz.concurrent.Task import scalaz.Scalaz._ class TestRepository(projects: Map[(Module, String), Project]) extends Repository { - def find(module: Module, version: String, cachePolicy: CachePolicy) = + val source = new core.Artifact.Source { + def artifacts(dependency: Dependency, project: Project) = ??? + } + def find(module: Module, version: String, cachePolicy: Repository.CachePolicy) = EitherT(Task.now( - projects.get((module, version)).toRightDisjunction("Not found") + projects.get((module, version)).map((source, _)).toRightDisjunction("Not found") )) - def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = ??? } diff --git a/core/src/test/scala/coursier/test/package.scala b/core/src/test/scala/coursier/test/package.scala index cf746c90e..863eb2519 100644 --- a/core/src/test/scala/coursier/test/package.scala +++ b/core/src/test/scala/coursier/test/package.scala @@ -1,26 +1,9 @@ package coursier -import scalaz.EitherT -import scalaz.concurrent.Task - package object test { implicit class DependencyOps(val underlying: Dependency) extends AnyVal { def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile) } - def resolve(dependencies: Set[Dependency], - fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], - maxIterations: Option[Int] = Some(200), - filter: Option[Dependency => Boolean] = None, - profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = { - - val startResolution = Resolution( - dependencies, - filter = filter, - profileActivation = profileActivation - ) - - startResolution.last(fetch, maxIterations.getOrElse(-1)) - } } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index 9d847c27e..ad707f46b 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -2,7 +2,7 @@ package coursier import java.net.{URI, URL} -import coursier.core.CachePolicy +import coursier.core.Repository.CachePolicy import scala.annotation.tailrec import scalaz.{-\/, \/-, \/, EitherT} diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index c6f15dc7c..476bc9d98 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,7 +1,7 @@ package coursier package web -import coursier.core.{DefaultFetchMetadata, Logger} +import coursier.core.{Repository, MavenRepository, Fetch} import japgolly.scalajs.react.vdom.{TagMod, Attr} import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope} @@ -18,7 +18,7 @@ case class ResolutionOptions(followOptional: Boolean = false, keepTest: Boolean = false) case class State(modules: Seq[Dependency], - repositories: Seq[MavenRepository[DefaultFetchMetadata]], + repositories: Seq[MavenRepository], options: ResolutionOptions, resolutionOpt: Option[Resolution], editModuleIdx: Int, @@ -71,7 +71,13 @@ class Backend($: BackendScope[Unit, State]) { def updateTree(resolution: Resolution, target: String, reverse: Boolean) = { def depsOf(dep: Dependency) = - resolution.projectCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter)) + resolution.projectCache + .get(dep.moduleVersion) + .toSeq + .flatMap{case (_, proj) => + core.Resolution.finalDependencies(dep, proj) + .filter(resolution.filter getOrElse core.Resolution.defaultFilter) + } val minDependencies = resolution.minDependencies @@ -105,7 +111,7 @@ class Backend($: BackendScope[Unit, State]) { g.$("#resLogTab a:last").tab("show") $.modState(_.copy(resolving = true, log = Nil)) - val logger: Logger = new Logger { + val logger: Fetch.Logger = new Fetch.Logger { def fetched(url: String) = { println(s"Fetched $url") $.modState(s => s.copy(log = s"Fetched $url" +: s.log)) @@ -128,7 +134,7 @@ class Backend($: BackendScope[Unit, State]) { ) ResolutionProcess(res) - .run(fetchSeveralFrom(s.repositories.map(r => r.copy(fetchMetadata = r.fetchMetadata.copy(logger = Some(logger))))), 100) + .run(Repository.fetchSeveralFrom(s.repositories.map(r => r.copy(fetch = r.fetch.copy(logger = Some(logger))))), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context @@ -239,8 +245,8 @@ object App { )), <.td(Seq[Seq[TagMod]]( res.projectCache.get(dep.moduleVersion) match { - case Some((MavenRepository(fetchMetadata, _), _)) => - // FIXME Maven specific, generalize if/when adding support for Ivy + case Some((source: MavenRepository.Source, proj)) if !source.ivyLike => + // FIXME Maven specific, generalize with source.artifacts val version0 = finalVersionOpt getOrElse dep.version val relPath = dep.module.organization.split('.').toSeq ++ Seq( @@ -249,11 +255,13 @@ object App { s"${dep.module.name}-$version0" ) + val root = source.root + Seq( - <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.pom", + <.a(^.href := s"$root${relPath.mkString("/")}.pom", <.span(^.`class` := "label label-info", "POM") ), - <.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.jar", + <.a(^.href := s"$root${relPath.mkString("/")}.jar", <.span(^.`class` := "label label-info", "JAR") ) ) @@ -387,19 +395,19 @@ object App { val modules = dependenciesTable("Dependencies") - val repositories = ReactComponentB[Seq[MavenRepository[DefaultFetchMetadata]]]("Repositories") + val repositories = ReactComponentB[Seq[MavenRepository]]("Repositories") .render{ repos => - def repoItem(repo: MavenRepository[DefaultFetchMetadata]) = + def repoItem(repo: MavenRepository) = <.tr( <.td( - <.a(^.href := repo.fetchMetadata.root, - repo.fetchMetadata.root + <.a(^.href := repo.fetch.root, + repo.fetch.root ) ) ) val sortedRepos = repos - .sortBy(repo => repo.fetchMetadata.root) + .sortBy(repo => repo.fetch.root) <.table(^.`class` := "table", <.thead( @@ -460,7 +468,7 @@ object App { } .build - val initialState = State(Nil, Seq(coursier.repository.mavenCentral), ResolutionOptions(), None, -1, resolving = false, reverseTree = false, log = Nil) + val initialState = State(Nil, Seq(Repository.mavenCentral), ResolutionOptions(), None, -1, resolving = false, reverseTree = false, log = Nil) val app = ReactComponentB[Unit]("Coursier") .initialState(initialState) From db3b244b755dcd5f1bdf3f90f18d2832ad8e2ae3 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 00:59:10 +0100 Subject: [PATCH 04/11] Add ~/.ivy2/local test --- .../scala/coursier/test/IvyLocalTests.scala | 18 ++++++++++++++++++ ...lexarchambault:coursier_2.11:0.1.0-SNAPSHOT | 7 +++++++ .../scala/coursier/test/CentralTests.scala | 10 ++++++---- 3 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 core-jvm/src/test/scala/coursier/test/IvyLocalTests.scala create mode 100644 core/src/test/resources/resolutions/com.github.alexarchambault:coursier_2.11:0.1.0-SNAPSHOT diff --git a/core-jvm/src/test/scala/coursier/test/IvyLocalTests.scala b/core-jvm/src/test/scala/coursier/test/IvyLocalTests.scala new file mode 100644 index 000000000..f9b820848 --- /dev/null +++ b/core-jvm/src/test/scala/coursier/test/IvyLocalTests.scala @@ -0,0 +1,18 @@ +package coursier.test + +import coursier.Module +import coursier.core.Repository +import utest._ + +object IvyLocalTests extends TestSuite { + + val tests = TestSuite{ + 'coursier{ + // Assume this module (and the sub-projects it depends on) is published locally + CentralTests.resolutionCheck( + Module("com.github.alexarchambault", "coursier_2.11"), "0.1.0-SNAPSHOT", + Some(Repository.ivy2Local)) + } + } + +} diff --git a/core/src/test/resources/resolutions/com.github.alexarchambault:coursier_2.11:0.1.0-SNAPSHOT b/core/src/test/resources/resolutions/com.github.alexarchambault:coursier_2.11:0.1.0-SNAPSHOT new file mode 100644 index 000000000..b7674ef1d --- /dev/null +++ b/core/src/test/resources/resolutions/com.github.alexarchambault:coursier_2.11:0.1.0-SNAPSHOT @@ -0,0 +1,7 @@ +com.github.alexarchambault:coursier_2.11:jar:0.1.0-SNAPSHOT +org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4 +org.scala-lang.modules:scala-xml_2.11:jar:1.0.4 +org.scala-lang:scala-library:jar:2.11.6 +org.scalaz:scalaz-concurrent_2.11:jar:7.1.2 +org.scalaz:scalaz-core_2.11:jar:7.1.2 +org.scalaz:scalaz-effect_2.11:jar:7.1.2 diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index bf0d4ae19..0d2b98148 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -13,21 +13,23 @@ object CentralTests extends TestSuite { Repository.mavenCentral ) - def resolve(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { + def resolve(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None, extraRepo: Option[Repository] = None) = { + val repositories0 = extraRepo.toSeq ++ repositories + ResolutionProcess(Resolution(deps, filter = filter)) - .run(Repository.fetchSeveralFrom(repositories)) + .run(Repository.fetchSeveralFrom(repositories0)) .runF } def repr(dep: Dependency) = s"${dep.module.organization}:${dep.module.name}:${dep.attributes.`type`}:${Some(dep.attributes.classifier).filter(_.nonEmpty).map(_+":").mkString}${dep.version}" - def resolutionCheck(module: Module, version: String) = + def resolutionCheck(module: Module, version: String, extraRepo: Option[Repository] = None) = async { val expected = await(textResource(s"resolutions/${module.organization}:${module.name}:$version")).split('\n').toSeq val dep = Dependency(module, version) - val res = await(resolve(Set(dep))) + val res = await(resolve(Set(dep), extraRepo = extraRepo)) val result = res.dependencies.toVector.map(repr).sorted.distinct From 7a10915ccff625a558bf3cbb585829d1edead910 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 00:59:11 +0100 Subject: [PATCH 05/11] Cleaning --- core/src/main/scala/coursier/core/Definitions.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/scala/coursier/core/Definitions.scala b/core/src/main/scala/coursier/core/Definitions.scala index 29f46793f..7e4b8bf93 100644 --- a/core/src/main/scala/coursier/core/Definitions.scala +++ b/core/src/main/scala/coursier/core/Definitions.scala @@ -70,7 +70,6 @@ case class Profile(id: String, dependencyManagement: Seq[Dependency], properties: Map[String, String]) -// FIXME Move to MavenRepository? case class Versions(latest: String, release: String, available: List[String], From dc2e2278b0bdb75d5ecc230a8726de0c38b51941 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 00:59:12 +0100 Subject: [PATCH 06/11] Various API changes --- .../main/scala/coursier/cli/Coursier.scala | 5 +- .../main/scala/coursier/core/Repository.scala | 9 --- .../coursier/core/ResolutionProcess.scala | 20 +++++- core/src/main/scala/coursier/package.scala | 62 ++++++++----------- .../scala/coursier/test/CentralTests.scala | 5 +- .../scala/coursier/test/PomParsingTests.scala | 13 ++-- .../scala/coursier/test/ResolutionTests.scala | 5 +- .../test/scala/coursier/test/package.scala | 27 ++++++++ web/src/main/scala/coursier/web/Backend.scala | 5 +- 9 files changed, 88 insertions(+), 63 deletions(-) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 6f2d2e34c..6349cb792 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -97,7 +97,7 @@ case class Coursier(scope: List[String], filter = Some(dep => (keepOptional || !dep.optional) && scopes(dep.scope)) ) - val fetchQuiet = Repository.fetchSeveralFrom(repositories) + val fetchQuiet = coursier.fetch(repositories) val fetch0 = if (verbose0 == 0) fetchQuiet else { @@ -109,7 +109,8 @@ case class Coursier(scope: List[String], print.flatMap(_ => fetchQuiet(modVers)) } - val res = ResolutionProcess(startRes) + val res = startRes + .process .run(fetch0, maxIterations) .run diff --git a/core/src/main/scala/coursier/core/Repository.scala b/core/src/main/scala/coursier/core/Repository.scala index 088b96961..54a3d1745 100644 --- a/core/src/main/scala/coursier/core/Repository.scala +++ b/core/src/main/scala/coursier/core/Repository.scala @@ -62,15 +62,6 @@ object Repository { } } - def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, Seq[String], (Artifact.Source, Project)] = - modVersion => find(repositories, modVersion._1, modVersion._2) - - def fetchSeveralFrom(repositories: Seq[Repository]): Seq[ModuleVersion] => Task[Seq[(ModuleVersion, Seq[String] \/ (Artifact.Source, Project))]] = { - val fetchOne = fetchFrom(repositories) - modVers => - Task.gatherUnordered(modVers.map(modVer => fetchOne(modVer).run.map(modVer -> _))) - } - sealed trait CachePolicy { def apply[E,T](local: => Task[E \/ T]) (remote: => Task[E \/ T]): Task[E \/ T] diff --git a/core/src/main/scala/coursier/core/ResolutionProcess.scala b/core/src/main/scala/coursier/core/ResolutionProcess.scala index adb4c27f8..147ce0ad0 100644 --- a/core/src/main/scala/coursier/core/ResolutionProcess.scala +++ b/core/src/main/scala/coursier/core/ResolutionProcess.scala @@ -11,13 +11,27 @@ sealed trait ResolutionProcess { (implicit F: Monad[F]): F[Resolution] = { if (maxIterations == 0) F.point(current) - else + else { + val maxIterations0 = if (maxIterations > 0) maxIterations - 1 else maxIterations + this match { case Done(res) => F.point(res) case missing0 @ Missing(missing, _, _) => - F.bind(fetch(missing))(result => missing0.next(result).run(fetch, if (maxIterations > 0) maxIterations - 1 else maxIterations)) - case cont @ Continue(_, _) => cont.nextNoCont.run(fetch) + F.bind(fetch(missing))(result => missing0.next(result).run(fetch, maxIterations0)) + case cont @ Continue(_, _) => cont.nextNoCont.run(fetch, maxIterations0) } + } + } + + def next[F[_]](fetch: ResolutionProcess.Fetch[F]) + (implicit F: Monad[F]): F[ResolutionProcess] = { + + this match { + case Done(res) => F.point(this) + case missing0 @ Missing(missing, _, _) => + F.map(fetch(missing))(result => missing0.next(result)) + case cont @ Continue(_, _) => cont.nextNoCont.next(fetch) + } } def current: Resolution diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index 96ecf6c4c..b0caddc55 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -1,6 +1,7 @@ +import scalaz.concurrent.Task /** - * Pulls definitions from coursier.core, with default arguments. + * Pulls definitions from coursier.core, sometimes with default arguments. */ package object coursier { @@ -23,34 +24,10 @@ package object coursier { } type Project = core.Project - object Project { - def apply(module: Module, - version: String, - dependencies: Seq[Dependency] = Seq.empty, - parent: Option[ModuleVersion] = None, - dependencyManagement: Seq[Dependency] = Seq.empty, - properties: Map[String, String] = Map.empty, - profiles: Seq[Profile] = Seq.empty, - versions: Option[core.Versions] = None): Project = - core.Project(module, version, dependencies, parent, dependencyManagement, properties, profiles, versions) - } + val Project: core.Project.type = core.Project type Profile = core.Profile - object Profile { - type Activation = core.Activation - object Activation { - def apply(properties: Seq[(String, Option[String])] = Nil): Activation = - core.Activation(properties) - } - - def apply(id: String, - activeByDefault: Option[Boolean] = None, - activation: Activation = Activation(), - dependencies: Seq[Dependency] = Nil, - dependencyManagement: Seq[Dependency] = Nil, - properties: Map[String, String] = Map.empty) = - core.Profile(id, activeByDefault, activation, dependencies, dependencyManagement, properties) - } + val Profile: core.Profile.type = core.Profile type Module = core.Module object Module { @@ -63,6 +40,9 @@ package object coursier { type Scope = core.Scope val Scope: core.Scope.type = core.Scope + type Repository = core.Repository + val Repository: core.Repository.type = core.Repository + type Resolution = core.Resolution object Resolution { val empty = apply() @@ -72,20 +52,30 @@ package object coursier { projectCache: Map[ModuleVersion, (Artifact.Source, Project)] = Map.empty, errorCache: Map[ModuleVersion, Seq[String]] = Map.empty, filter: Option[Dependency => Boolean] = None, - profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution = + profileActivation: Option[(String, core.Activation, Map[String, String]) => Boolean] = None): Resolution = core.Resolution(rootDependencies, dependencies, conflicts, projectCache, errorCache, filter, profileActivation) } type Artifact = core.Artifact - object Artifact { - def apply(url: String, - extra: Map[String, String] = Map.empty, - attributes: Attributes = Attributes()): Artifact = - core.Artifact(url, extra, attributes) - - type Source = core.Artifact.Source - } + val Artifact: core.Artifact.type = core.Artifact type ResolutionProcess = core.ResolutionProcess val ResolutionProcess: core.ResolutionProcess.type = core.ResolutionProcess + + implicit class ResolutionExtensions(val underlying: Resolution) extends AnyVal { + def process: ResolutionProcess = ResolutionProcess(underlying) + } + + def fetch(repositories: Seq[core.Repository]): ResolutionProcess.Fetch[Task] = { + modVers => + Task.gatherUnordered( + modVers + .map(modVer => + Repository.find(repositories, modVer._1, modVer._2) + .run + .map(modVer -> _) + ) + ) + } + } diff --git a/core/src/test/scala/coursier/test/CentralTests.scala b/core/src/test/scala/coursier/test/CentralTests.scala index 0d2b98148..10e7cb5b4 100644 --- a/core/src/test/scala/coursier/test/CentralTests.scala +++ b/core/src/test/scala/coursier/test/CentralTests.scala @@ -16,8 +16,9 @@ object CentralTests extends TestSuite { def resolve(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None, extraRepo: Option[Repository] = None) = { val repositories0 = extraRepo.toSeq ++ repositories - ResolutionProcess(Resolution(deps, filter = filter)) - .run(Repository.fetchSeveralFrom(repositories0)) + Resolution(deps, filter = filter) + .process + .run(fetch(repositories0)) .runF } diff --git a/core/src/test/scala/coursier/test/PomParsingTests.scala b/core/src/test/scala/coursier/test/PomParsingTests.scala index 3135c4905..ae56eab8a 100644 --- a/core/src/test/scala/coursier/test/PomParsingTests.scala +++ b/core/src/test/scala/coursier/test/PomParsingTests.scala @@ -5,7 +5,6 @@ import utest._ import scalaz._ import coursier.core.Xml -import coursier.Profile.Activation import coursier.core.compatibility._ object PomParsingTests extends TestSuite { @@ -34,7 +33,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Profile("profile1", None, Activation(Nil), Nil, Nil, Map.empty)) + val expected = \/-(Profile("profile1", None, Profile.Activation(Nil), Nil, Nil, Map.empty)) val result = Xml.profile(xmlParse(profileNode).right.get) @@ -49,7 +48,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Profile("", Some(true), Activation(Nil), Nil, Nil, Map.empty)) + val expected = \/-(Profile("", Some(true), Profile.Activation(Nil), Nil, Nil, Map.empty)) val result = Xml.profile(xmlParse(profileNode).right.get) @@ -65,7 +64,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Profile("profile1", Some(true), Activation(Nil), Nil, Nil, Map.empty)) + val expected = \/-(Profile("profile1", Some(true), Profile.Activation(Nil), Nil, Nil, Map.empty)) val result = Xml.profile(xmlParse(profileNode).right.get) @@ -88,7 +87,7 @@ object PomParsingTests extends TestSuite { val expected = \/-(Profile( "profile1", None, - Activation(Nil), + Profile.Activation(Nil), Seq( Dependency(Module("comp", "lib"), "0.2")), Nil, @@ -119,7 +118,7 @@ object PomParsingTests extends TestSuite { val expected = \/-(Profile( "profile1", None, - Activation(Nil), + Profile.Activation(Nil), Nil, Seq( Dependency(Module("comp", "lib"), "0.2", scope = Scope.Test)), @@ -143,7 +142,7 @@ object PomParsingTests extends TestSuite { val expected = \/-(Profile( "profile1", None, - Activation(Nil), + Profile.Activation(Nil), Nil, Nil, Map("first.prop" -> "value1") diff --git a/core/src/test/scala/coursier/test/ResolutionTests.scala b/core/src/test/scala/coursier/test/ResolutionTests.scala index 66f7f20d1..c88266e18 100644 --- a/core/src/test/scala/coursier/test/ResolutionTests.scala +++ b/core/src/test/scala/coursier/test/ResolutionTests.scala @@ -10,8 +10,9 @@ import coursier.test.compatibility._ object ResolutionTests extends TestSuite { def resolve0(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { - ResolutionProcess(Resolution(deps, filter = filter)) - .run(Repository.fetchSeveralFrom(repositories)) + Resolution(deps, filter = filter) + .process + .run(fetch(repositories)) .runF } diff --git a/core/src/test/scala/coursier/test/package.scala b/core/src/test/scala/coursier/test/package.scala index 863eb2519..b2feafef9 100644 --- a/core/src/test/scala/coursier/test/package.scala +++ b/core/src/test/scala/coursier/test/package.scala @@ -6,4 +6,31 @@ package object test { def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile) } + object Profile { + type Activation = core.Activation + object Activation { + def apply(properties: Seq[(String, Option[String])] = Nil): Activation = + core.Activation(properties) + } + + def apply(id: String, + activeByDefault: Option[Boolean] = None, + activation: Activation = Activation(), + dependencies: Seq[Dependency] = Nil, + dependencyManagement: Seq[Dependency] = Nil, + properties: Map[String, String] = Map.empty) = + core.Profile(id, activeByDefault, activation, dependencies, dependencyManagement, properties) + } + + object Project { + def apply(module: Module, + version: String, + dependencies: Seq[Dependency] = Seq.empty, + parent: Option[ModuleVersion] = None, + dependencyManagement: Seq[Dependency] = Seq.empty, + properties: Map[String, String] = Map.empty, + profiles: Seq[Profile] = Seq.empty, + versions: Option[core.Versions] = None): Project = + core.Project(module, version, dependencies, parent, dependencyManagement, properties, profiles, versions) + } } diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 476bc9d98..4c8c87bf5 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -133,8 +133,9 @@ class Backend($: BackendScope[Unit, State]) { filter = Some(dep => (s.options.followOptional || !dep.optional) && (s.options.keepTest || dep.scope != Scope.Test)) ) - ResolutionProcess(res) - .run(Repository.fetchSeveralFrom(s.repositories.map(r => r.copy(fetch = r.fetch.copy(logger = Some(logger))))), 100) + res + .process + .run(fetch(s.repositories.map(r => r.copy(fetch = r.fetch.copy(logger = Some(logger))))), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context From dc6e1c368c8954808feafab7c0955365b509d8f7 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 00:59:13 +0100 Subject: [PATCH 07/11] Minor API change --- core/src/main/scala/coursier/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/coursier/package.scala b/core/src/main/scala/coursier/package.scala index b0caddc55..c50456072 100644 --- a/core/src/main/scala/coursier/package.scala +++ b/core/src/main/scala/coursier/package.scala @@ -66,7 +66,7 @@ package object coursier { def process: ResolutionProcess = ResolutionProcess(underlying) } - def fetch(repositories: Seq[core.Repository]): ResolutionProcess.Fetch[Task] = { + implicit def fetch(repositories: Seq[core.Repository]): ResolutionProcess.Fetch[Task] = { modVers => Task.gatherUnordered( modVers From 1743bbbf1d133e0d5167a0f0e82286d36b7bbfba Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 02:44:25 +0200 Subject: [PATCH 08/11] Change in build definition Hopefully friendlier to SBT on Travis (why tf is it perfectly fine locally then??? Same JDK, same SBT version, ...) --- project/Coursier.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/project/Coursier.scala b/project/Coursier.scala index d11e0f0c7..a530be5b5 100644 --- a/project/Coursier.scala +++ b/project/Coursier.scala @@ -151,4 +151,14 @@ object CoursierBuild extends Build { ) .enablePlugins(ScalaJSPlugin) + lazy val root = Project(id = "root", base = file(".")) + .aggregate(coreJvm, coreJs, files, cli, web) + .settings(commonSettings: _*) + .settings( + (unmanagedSourceDirectories in Compile) := Nil, + (unmanagedSourceDirectories in Test) := Nil, + publish := (), + publishLocal := () + ) + } From 0296322d01c5f885e2f77ff91a8785d52747e72c Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 02:54:39 +0200 Subject: [PATCH 09/11] Update Travis script --- project/travis.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/project/travis.sh b/project/travis.sh index acb7f8a2b..47fbbeff5 100755 --- a/project/travis.sh +++ b/project/travis.sh @@ -34,6 +34,7 @@ else SBT_COMMANDS="compile" fi +SBT_COMMANDS="$SBT_COMMANDS publish-local" # Required for ~/.ivy2/local repo tests SBT_COMMANDS="$SBT_COMMANDS core-jvm/test core-js/test" PUSH_GHPAGES=0 From 6f3384c260dc836f360508c9ec69163a80858b99 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 03:11:56 +0200 Subject: [PATCH 10/11] Update USAGE.md --- USAGE.md | 48 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/USAGE.md b/USAGE.md index a0357fde2..c7a47379a 100644 --- a/USAGE.md +++ b/USAGE.md @@ -11,8 +11,11 @@ libraryDependencies += Then, ```scala import coursier._ + +// Cache for metadata can be setup here, see cli/src/main/scala/coursier/Coursier.scala val repositories = Seq( - repository.mavenCentral + Repository.ivy2Local, + Repository.mavenCentral ) val dependencies = Set( @@ -22,23 +25,40 @@ val dependencies = Set( val resolution = - resolve(dependencies, fetchFrom(repositories)).run + Resolution(dependencies) + .process + .run(repositories) + .run + +// Note that only metadata are downloaded during resolution assert(resolution.isDone) // Check that resolution converged -// Printing the results -for (dep <- resolution.dependencies if resolution.projectsCache.contains(dep.moduleVersion)) - println(resolution.projectsCache(dep.moduleVersion)) -for (dep <- resolution.dependencies if resolution.errors.contains(dep.moduleVersion)) - println(resolution.errors(dep.moduleVersion)) +// Errors in +resolution.errors // Seq[(Dependency, Seq[String])], the Seq[String] contains the errors returned by each repository -// Downloading them -import coursier.core.ArtifactDownloader +// Artifact URLs in +resolution.artifacts // Seq[Artifact] +// Artifact has in particular a field url: String -val dl = ArtifactDownloader(repository.mavenCentral.root, new java.io.File("cache")) -for (dep <- resolution.dependencies if resolution.projectsCache.contains(dep.moduleVersion)) - dl.artifact(dep).run.run match { - case -\/(err) => println(s"Failed to download ${dep.moduleVersion}: $err") - case \/-(file) => println(s"${dep.moduleVersion}: $file") + +// Now if we want to download or cache the artifacts, add in build.sbt +// "com.github.alexarchambault" %% "coursier-files" % "0.1.0-SNAPSHOT" + +val files = Files( + Seq(), + () => ???, // TODO Tmp directory for URLs with no cache + Some(logger) // Optional, logger: FilesLogger +) + +val cachePolicy = Repository.CachePolicy.Default + +for (artifact <- artifacts) + files.file(artifact, cachePolicy).run match { + case -\/(err) => // Download failed, err: String + case \/-(file) => // Success, file: java.io.File } + +// Artifacts can be downloaded in parallel thanks to Task.gatherUnordered +// See the example in cli/src/main/scala/coursier/Coursier.scala ``` From a1bb46d6c464281b436ce8129c737ec77fbcd4ca Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 26 Jun 2015 03:12:13 +0200 Subject: [PATCH 11/11] Fix Travis script (2.10 build) --- project/travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/travis.sh b/project/travis.sh index 47fbbeff5..6158cfa21 100755 --- a/project/travis.sh +++ b/project/travis.sh @@ -34,7 +34,7 @@ else SBT_COMMANDS="compile" fi -SBT_COMMANDS="$SBT_COMMANDS publish-local" # Required for ~/.ivy2/local repo tests +sbt ++2.11.6 core-jvm/publish-local # Required for ~/.ivy2/local repo tests SBT_COMMANDS="$SBT_COMMANDS core-jvm/test core-js/test" PUSH_GHPAGES=0