luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rework Resolution iteration and artifacts API

This commit is contained in:
Alexandre Archambault 2015-06-26 00:59:08 +01:00
parent 6a91ddfaca
commit b010d4b620
18 changed files with 308 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
cli/src/main/scala/coursier/cli/Coursier.scala
View File

@ -4,15 +4,14 @@ package cli
import java.io.File
import caseapp._
import coursier.core.{CachePolicy, Parse}
import coursier.core.MetadataFetchLogger
import coursier.core.{Fetch, Parse, Repository}, Repository.CachePolicy
import scalaz.concurrent.Task
case class Coursier(scope: List[String],
keepOptional: Boolean,
fetch: Boolean,
verbose: List[Unit],
@ExtraName("v") verbose: List[Unit],
@ExtraName("N") maxIterations: Int = 100) extends App {
val verbose0 = verbose.length
@ -27,48 +26,45 @@ case class Coursier(scope: List[String],
def fileRepr(f: File) = f.toString
val logger: Option[MetadataFetchLogger with FilesLogger] =
if (verbose0 <= 1) None
else Some(
new MetadataFetchLogger with FilesLogger {
def println(s: String) = Console.err.println(s)
val logger: Fetch.Logger with FilesLogger =
new Fetch.Logger with FilesLogger {
def println(s: String) = Console.err.println(s)
def downloading(url: String) =
println(s"Downloading $url")
def downloaded(url: String, success: Boolean) =
println(
if (success) s"Downloaded $url"
else s"Failed to download $url"
)
def readingFromCache(f: File) = {
println(s"Reading ${fileRepr(f)} from cache")
}
def puttingInCache(f: File) =
println(s"Writing ${fileRepr(f)} in cache")
def foundLocally(f: File) =
println(s"Found locally ${fileRepr(f)}")
def downloadingArtifact(url: String) =
println(s"Downloading $url")
def downloadedArtifact(url: String, success: Boolean) =
println(
if (success) s"Downloaded $url"
else s"Failed to download $url"
)
def downloading(url: String) =
println(s"Downloading $url")
def downloaded(url: String, success: Boolean) =
println(
if (success) s"Downloaded $url"
else s"Failed to download $url"
)
def readingFromCache(f: File) = {
println(s"Reading ${fileRepr(f)} from cache")
}
)
def puttingInCache(f: File) =
println(s"Writing ${fileRepr(f)} in cache")
val cachedMavenCentral = repository.mavenCentral.copy(
fetchMetadata = repository.mavenCentral.fetchMetadata.copy(
def foundLocally(f: File) =
println(s"Found locally ${fileRepr(f)}")
def downloadingArtifact(url: String) =
println(s"Downloading $url")
def downloadedArtifact(url: String, success: Boolean) =
println(
if (success) s"Downloaded $url"
else s"Failed to download $url"
)
}
val cachedMavenCentral = Repository.mavenCentral.copy(
fetch = Repository.mavenCentral.fetch.copy(
cache = Some(centralCacheDir),
logger = logger
logger = if (verbose0 <= 1) None else Some(logger)
)
)
val repositories = Seq[Repository](
cachedMavenCentral,
repository.ivy2Local.copy(
fetchMetadata = repository.ivy2Local.fetchMetadata.copy(
logger = logger
Repository.ivy2Local.copy(
fetch = Repository.ivy2Local.fetch.copy(
logger = if (verbose0 <= 1) None else Some(logger)
)
)
)
@ -101,9 +97,9 @@ case class Coursier(scope: List[String],
filter = Some(dep => (keepOptional || !dep.optional) && scopes(dep.scope))
)
val fetchQuiet = fetchSeveralFrom(repositories)
val fetchQuiet = Repository.fetchSeveralFrom(repositories)
val fetch0 =
if (verbose == 0) fetchQuiet
if (verbose0 == 0) fetchQuiet
else {
modVers: Seq[(Module, String)] =>
val print = Task{
@ -158,10 +154,10 @@ case class Coursier(scope: List[String],
val files = new Files(
Seq(
cachedMavenCentral.fetchMetadata.root -> centralFilesCacheDir
cachedMavenCentral.fetch.root -> centralFilesCacheDir
),
() => ???,
logger
if (verbose0 <= 0) None else Some(logger)
)
val tasks = artifacts.map(files.file(_, cachePolicy).run)

22
core-js/src/main/scala/coursier/core/DefaultFetchMetadata.scala → core-js/src/main/scala/coursier/core/Fetch.scala
View File

@ -12,7 +12,7 @@ import js.Dynamic.{global => g}
import scala.scalajs.js.timers._
object DefaultFetchMetadata {
object Fetch {
def encodeURIComponent(s: String): String =
g.encodeURIComponent(s).asInstanceOf[String]
@ -70,25 +70,27 @@ object DefaultFetchMetadata {
p.future
}
trait Logger {
def fetching(url: String): Unit
def fetched(url: String): Unit
def other(url: String, msg: String): Unit
}
}
trait Logger {
def fetching(url: String): Unit
def fetched(url: String): Unit
def other(url: String, msg: String): Unit
}
case class Fetch(root: String,
logger: Option[Fetch.Logger] = None) {
case class DefaultFetchMetadata(root: String,
logger: Option[Logger] = None) extends FetchMetadata {
def apply(artifact: Artifact,
cachePolicy: CachePolicy): EitherT[Task, String, String] = {
cachePolicy: Repository.CachePolicy): EitherT[Task, String, String] = {
EitherT(
Task { implicit ec =>
DefaultFetchMetadata.get(root + artifact.url)
Fetch.get(root + artifact.url)
.map(\/-(_))
.recover{case e: Exception => -\/(e.getMessage)}
}
)
}
}

10
core-js/src/test/scala/coursier/test/JsTests.scala
View File

@ -1,7 +1,7 @@
package coursier
package test
import coursier.core.DefaultFetchMetadata
import coursier.core.{Fetch, Repository}
import coursier.test.compatibility._
import utest._
@ -11,14 +11,14 @@ import scala.concurrent.{Future, Promise}
object JsTests extends TestSuite {
val tests = TestSuite {
'promise {
'promise{
val p = Promise[Unit]()
Future(p.success(()))
p.future
}
'get{
DefaultFetchMetadata.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom")
Fetch.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom")
.map(core.compatibility.xmlParse)
.map{ xml =>
assert(xml.right.toOption.exists(_.label == "project"))
@ -26,9 +26,9 @@ object JsTests extends TestSuite {
}
'getProj{
repository.mavenCentral
Repository.mavenCentral
.find(Module("ch.qos.logback", "logback-classic"), "1.1.3")
.map{ proj =>
.map{case (_, proj) =>
assert(proj.parent == Some(Module("ch.qos.logback", "logback-parent"), "1.1.3"))
}
.run

28
core-jvm/src/main/scala/coursier/core/DefaultFetchMetadata.scala → core-jvm/src/main/scala/coursier/core/Fetch.scala
View File

@ -8,20 +8,15 @@ import scala.io.Codec
import scalaz._, Scalaz._
import scalaz.concurrent.Task
trait MetadataFetchLogger {
def downloading(url: String): Unit
def downloaded(url: String, success: Boolean): Unit
def readingFromCache(f: File): Unit
def puttingInCache(f: File): Unit
}
case class DefaultFetchMetadata(root: String,
cache: Option[File] = None,
logger: Option[MetadataFetchLogger] = None) extends FetchMetadata {
case class Fetch(root: String,
cache: Option[File] = None,
logger: Option[Fetch.Logger] = None) {
val isLocal = root.startsWith("file:///")
def apply(artifact: Artifact, cachePolicy: CachePolicy): EitherT[Task, String, String] = {
def apply(artifact: Artifact,
cachePolicy: Repository.CachePolicy): EitherT[Task, String, String] = {
def locally(eitherFile: String \/ File) = {
Task {
for {
@ -49,7 +44,7 @@ case class DefaultFetchMetadata(root: String,
val url = new URL(urlStr)
def log = Task(logger.foreach(_.downloading(urlStr)))
def get = DefaultFetchMetadata.readFully(url.openStream())
def get = Fetch.readFully(url.openStream())
log.flatMap(_ => get)
}
@ -75,7 +70,14 @@ case class DefaultFetchMetadata(root: String,
}
object DefaultFetchMetadata {
object Fetch {
trait Logger {
def downloading(url: String): Unit
def downloaded(url: String, success: Boolean): Unit
def readingFromCache(f: File): Unit
def puttingInCache(f: File): Unit
}
def readFullySync(is: InputStream) = {
val buffer = new ByteArrayOutputStream()

31
core-jvm/src/main/scala/coursier/core/package.scala
View File

@ -1,31 +0,0 @@
package coursier
import scalaz.EitherT
import scalaz.concurrent.Task
package object core {
def resolution(dependencies: Set[Dependency],
fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)],
filter: Option[Dependency => Boolean],
profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]): Stream[Resolution] = {
val startResolution = Resolution(
dependencies, Set.empty, Set.empty,
Map.empty, Map.empty,
filter,
profileActivation
)
def helper(resolution: Resolution): Stream[Resolution] = {
if (resolution.isDone) Stream()
else {
val nextRes = resolution.next(fetch).run
nextRes #:: helper(nextRes)
}
}
startResolution #:: helper(startResolution)
}
}

4
core-jvm/src/test/scala/coursier/test/compatibility/package.scala
View File

@ -1,6 +1,6 @@
package coursier.test
import coursier.core.DefaultFetchMetadata
import coursier.core.Fetch
import scala.concurrent.{ExecutionContext, Future}
import scalaz.concurrent.Task
@ -17,7 +17,7 @@ package object compatibility {
def is = getClass.getClassLoader
.getResource(path).openStream()
new String(DefaultFetchMetadata.readFullySync(is), "UTF-8")
new String(Fetch.readFullySync(is), "UTF-8")
}
}

5
core/src/main/scala/coursier/core/Definitions.scala
View File

@ -102,4 +102,9 @@ object Artifact {
val javadocSig = "javadoc-pgp"
val javadocSigMd5 = "md5-javadoc-pgp"
val javadocSigSha1 = "sha1-javadoc-pgp"
trait Source {
def artifacts(dependency: Dependency,
project: Project): Seq[Artifact]
}
}

283
core/src/main/scala/coursier/core/Repository.scala
View File

@ -1,45 +1,113 @@
package coursier.core
import coursier.core.Resolution.ModuleVersion
import scalaz.{-\/, \/-, \/, EitherT}
import scalaz.concurrent.Task
import coursier.core.compatibility.encodeURIComponent
trait Repository {
def find(module: Module, version: String, cachePolicy: CachePolicy = CachePolicy.Default): EitherT[Task, String, Project]
def artifacts(dependency: Dependency, project: Project): Seq[Artifact]
}
sealed trait CachePolicy {
def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T]
def saving[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T])(save: => T => Task[Unit]): Task[E \/ T] =
apply(local)(CachePolicy.saving(remote)(save))
}
object CachePolicy {
def saving[E,T](remote: => Task[E \/ T])(save: T => Task[Unit]): Task[E \/ T] = {
for {
res <- remote
_ <- res.fold(_ => Task.now(()), t => save(t))
} yield res
}
case object Default extends CachePolicy {
def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T] =
local.flatMap(res => if (res.isLeft) remote else Task.now(res))
}
case object LocalOnly extends CachePolicy {
def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T] =
local
}
case object ForceDownload extends CachePolicy {
def apply[E,T](local: => Task[E \/ T])(remote: => Task[E \/ T]): Task[E \/ T] =
remote
}
def find(module: Module,
version: String,
cachePolicy: Repository.CachePolicy = Repository.CachePolicy.Default): EitherT[Task, String, (Artifact.Source, Project)]
}
object Repository {
val mavenCentral = MavenRepository(Fetch("https://repo1.maven.org/maven2/"))
val sonatypeReleases = MavenRepository(Fetch("https://oss.sonatype.org/content/repositories/releases/"))
val sonatypeSnapshots = MavenRepository(Fetch("https://oss.sonatype.org/content/repositories/snapshots/"))
lazy val ivy2Local = MavenRepository(Fetch("file://" + sys.props("user.home") + "/.ivy2/local/"), ivyLike = true)
/**
* Try to find `module` among `repositories`.
*
* Look at `repositories` from the left, one-by-one, and stop at first success.
* Else, return all errors, in the same order.
*
* The `version` field of the returned `Project` in case of success may not be
* equal to the provided one, in case the latter is not a specific
* version (e.g. version interval). Which version get chosen depends on
* the repository implementation.
*/
def find(repositories: Seq[Repository],
module: Module,
version: String): EitherT[Task, Seq[String], (Artifact.Source, Project)] = {
val lookups = repositories.map(repo => repo -> repo.find(module, version).run)
val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Artifact.Source, Project)]) {
case (acc, (repo, t)) =>
acc.flatMap {
case -\/(errors) =>
t.map(res => res
.flatMap{case (source, project) =>
if (project.module == module) \/-((source, project))
else -\/(s"Wrong module returned (expected: $module, got: ${project.module})")
}
.leftMap(error => error +: errors)
)
case res @ \/-(_) =>
Task.now(res)
}
}
EitherT(task.map(_.leftMap(_.reverse))).map {case x @ (_, proj) =>
assert(proj.module == module)
x
}
}
def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, Seq[String], (Artifact.Source, Project)] =
modVersion => find(repositories, modVersion._1, modVersion._2)
def fetchSeveralFrom(repositories: Seq[Repository]): Seq[ModuleVersion] => Task[Seq[(ModuleVersion, Seq[String] \/ (Artifact.Source, Project))]] = {
val fetchOne = fetchFrom(repositories)
modVers =>
Task.gatherUnordered(modVers.map(modVer => fetchOne(modVer).run.map(modVer -> _)))
}
sealed trait CachePolicy {
def apply[E,T](local: => Task[E \/ T])
(remote: => Task[E \/ T]): Task[E \/ T]
def saving[E,T](local: => Task[E \/ T])
(remote: => Task[E \/ T])
(save: => T => Task[Unit]): Task[E \/ T] =
apply(local)(CachePolicy.saving(remote)(save))
}
object CachePolicy {
def saving[E,T](remote: => Task[E \/ T])
(save: T => Task[Unit]): Task[E \/ T] = {
for {
res <- remote
_ <- res.fold(_ => Task.now(()), t => save(t))
} yield res
}
case object Default extends CachePolicy {
def apply[E,T](local: => Task[E \/ T])
(remote: => Task[E \/ T]): Task[E \/ T] =
local
.flatMap(res => if (res.isLeft) remote else Task.now(res))
}
case object LocalOnly extends CachePolicy {
def apply[E,T](local: => Task[E \/ T])
(remote: => Task[E \/ T]): Task[E \/ T] =
local
}
case object ForceDownload extends CachePolicy {
def apply[E,T](local: => Task[E \/ T])
(remote: => Task[E \/ T]): Task[E \/ T] =
remote
}
}
implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal {
def withDefaultChecksums: Artifact =
underlying.copy(extra = underlying.extra ++ Seq(
@ -72,24 +140,88 @@ object Repository {
}
}
trait FetchMetadata {
def root: String
def apply(artifact: Artifact,
cachePolicy: CachePolicy): EitherT[Task, String, String]
object MavenRepository {
def ivyLikePath(org: String,
name: String,
version: String,
subDir: String,
baseSuffix: String,
ext: String) =
Seq(
org,
name,
version,
subDir,
s"$name$baseSuffix.$ext"
)
case class Source(root: String, ivyLike: Boolean) extends Artifact.Source {
import Repository._
def artifacts(dependency: Dependency,
project: Project): Seq[Artifact] = {
def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) =
MavenRepository.ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext)
val path =
if (ivyLike)
ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`)
else
dependency.module.organization.split('.').toSeq ++ Seq(
dependency.module.name,
project.version,
s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}"
)
var artifact =
Artifact(
root + path.mkString("/"),
Map.empty,
dependency.attributes
)
.withDefaultChecksums
if (dependency.attributes.`type` == "jar") {
artifact = artifact.withDefaultSignature
artifact =
if (ivyLike) {
val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/")
val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/")
artifact
.copy(extra = artifact.extra ++ Map(
Artifact.sourcesMd5 -> (srcPath + ".md5"),
Artifact.sourcesSha1 -> (srcPath + ".sha1"),
Artifact.sources -> srcPath,
Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"),
Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"),
Artifact.sourcesSig -> (srcPath + ".asc"),
Artifact.javadocMd5 -> (javadocPath + ".md5"),
Artifact.javadocSha1 -> (javadocPath + ".sha1"),
Artifact.javadoc -> javadocPath,
Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"),
Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"),
Artifact.javadocSig -> (javadocPath + ".asc")
))
} else artifact.withJavadocSources
}
Seq(artifact)
}
}
}
case class MavenRepository[F <: FetchMetadata](fetchMetadata: F,
ivyLike: Boolean = false) extends Repository {
case class MavenRepository(fetch: Fetch,
ivyLike: Boolean = false) extends Repository {
import Repository._
import MavenRepository._
def ivyLikePath(org: String, name: String, version: String, subDir: String, baseSuffix: String, ext: String) = Seq(
org,
name,
version,
subDir,
s"$name$baseSuffix.$ext"
)
val source = MavenRepository.Source(fetch.root, ivyLike)
def projectArtifact(module: Module, version: String): Artifact = {
val path = (
@ -142,7 +274,7 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F,
versionsArtifact(module) match {
case None => Task.now(-\/("Not supported"))
case Some(artifact) =>
fetchMetadata(artifact, cachePolicy)
fetch(artifact, cachePolicy)
.run
.map(eitherStr =>
for {
@ -161,7 +293,7 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F,
cachePolicy: CachePolicy): EitherT[Task, String, Project] = {
EitherT {
fetchMetadata(projectArtifact(module, version), cachePolicy)
fetch(projectArtifact(module, version), cachePolicy)
.run
.map(eitherStr =>
for {
@ -176,10 +308,10 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F,
def find(module: Module,
version: String,
cachePolicy: CachePolicy): EitherT[Task, String, Project] = {
cachePolicy: CachePolicy): EitherT[Task, String, (Artifact.Source, Project)] = {
Parse.versionInterval(version).filter(_.isValid) match {
case None => findNoInterval(module, version, cachePolicy)
case None => findNoInterval(module, version, cachePolicy).map((source, _))
case Some(itv) =>
versions(module, cachePolicy)
.flatMap { versions0 =>
@ -198,65 +330,14 @@ case class MavenRepository[F <: FetchMetadata](fetchMetadata: F,
}
eitherVersion match {
case -\/(reason) => EitherT[Task, String, Project](Task.now(-\/(reason)))
case -\/(reason) => EitherT[Task, String, (Artifact.Source, Project)](Task.now(-\/(reason)))
case \/-(version0) =>
findNoInterval(module, version0, cachePolicy)
.map(_.copy(versions = Some(versions0)))
.map((source, _))
}
}
}
}
def artifacts(dependency: Dependency,
project: Project): Seq[Artifact] = {
def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) =
ivyLikePath(dependency.module.organization, dependency.module.name, project.version, subDir, baseSuffix, ext)
val path =
if (ivyLike)
ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`)
else
dependency.module.organization.split('.').toSeq ++ Seq(
dependency.module.name,
project.version,
s"${dependency.module.name}-${project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}"
)
var artifact =
Artifact(
fetchMetadata.root + path.mkString("/"),
Map.empty,
dependency.attributes
)
.withDefaultChecksums
if (dependency.attributes.`type` == "jar") {
artifact = artifact.withDefaultSignature
artifact =
if (ivyLike) {
val srcPath = fetchMetadata.root + ivyLikePath0("srcs", "-sources", "jar").mkString("/")
val javadocPath = fetchMetadata.root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/")
artifact
.copy(extra = artifact.extra ++ Map(
Artifact.sourcesMd5 -> (srcPath + ".md5"),
Artifact.sourcesSha1 -> (srcPath + ".sha1"),
Artifact.sources -> srcPath,
Artifact.sourcesSigMd5 -> (srcPath + ".asc.md5"),
Artifact.sourcesSigSha1 -> (srcPath + ".asc.sha1"),
Artifact.sourcesSig -> (srcPath + ".asc"),
Artifact.javadocMd5 -> (javadocPath + ".md5"),
Artifact.javadocSha1 -> (javadocPath + ".sha1"),
Artifact.javadoc -> javadocPath,
Artifact.javadocSigMd5 -> (javadocPath + ".asc.md5"),
Artifact.javadocSigSha1 -> (javadocPath + ".asc.sha1"),
Artifact.javadocSig -> (javadocPath + ".asc")
))
} else artifact.withJavadocSources
}
Seq(artifact)
}
}

99
core/src/main/scala/coursier/core/Resolution.scala
View File

@ -4,52 +4,12 @@ import java.util.regex.Pattern.quote
import scala.annotation.tailrec
import scala.collection.mutable
import scalaz.concurrent.Task
import scalaz.{EitherT, \/-, \/, -\/}
import scalaz.{\/-, -\/}
object Resolution {
type ModuleVersion = (Module, String)
/**
* Try to find `module` among `repositories`.
*
* Look at `repositories` from the left, one-by-one, and stop at first success.
* Else, return all errors, in the same order.
*
* The `version` field of the returned `Project` in case of success may not be
* equal to the provided one, in case the latter is not a specific
* version (e.g. version interval). Which version get chosen depends on
* the repository implementation.
*/
def find(repositories: Seq[Repository],
module: Module,
version: String): EitherT[Task, Seq[String], (Repository, Project)] = {
val lookups = repositories.map(repo => repo -> repo.find(module, version).run)
val task = lookups.foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Repository, Project)]) {
case (acc, (repo, t)) =>
acc.flatMap {
case -\/(errors) =>
t.map(res => res
.flatMap(project =>
if (project.module == module) \/-((repo, project))
else -\/(s"Wrong module returned (expected: $module, got: ${project.module})")
)
.leftMap(error => error +: errors)
)
case res @ \/-(_) =>
Task.now(res)
}
}
EitherT(task.map(_.leftMap(_.reverse))).map { case x @ (_, proj) =>
assert(proj.module == module)
x
}
}
/**
* Get the active profiles of `project`, using the current properties `properties`,
* and `profileActivation` stating if a profile is active.
@ -328,7 +288,7 @@ object Resolution {
case class Resolution(rootDependencies: Set[Dependency],
dependencies: Set[Dependency],
conflicts: Set[Dependency],
projectCache: Map[Resolution.ModuleVersion, (Repository, Project)],
projectCache: Map[Resolution.ModuleVersion, (Artifact.Source, Project)],
errorCache: Map[Resolution.ModuleVersion, Seq[String]],
filter: Option[Dependency => Boolean],
profileActivation: Option[(String, Activation, Map[String, String]) => Boolean]) {
@ -466,15 +426,6 @@ case class Resolution(rootDependencies: Set[Dependency],
} else this
}
/**
* Do a new iteration, fetching the missing modules along the way.
*/
def next(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)]): Task[Resolution] = {
val missing = missingFromCache
if (missing.isEmpty) Task.now(nextNoMissingUnsafe)
else fetch(missing.toList, fetchModule).map(_.nextIfNoMissing)
}
/**
* Required modules for the dependency management of `project`.
*/
@ -575,56 +526,14 @@ case class Resolution(rootDependencies: Set[Dependency],
)
}
/**
* Fetch `modules` with `fetchModules`, and add the resulting errors and projects to the cache.
*/
def fetch(modules: Seq[ModuleVersion],
fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)]): Task[Resolution] = {
val lookups = modules.map(dep => fetchModule(dep).run.map(dep -> _))
val gatheredLookups = Task.gatherUnordered(lookups, exceptionCancels = true)
gatheredLookups.flatMap{ lookupResults =>
val errors0 = errorCache ++ lookupResults.collect{case (modVer, -\/(repoErrors)) => modVer -> repoErrors}
val newProjects = lookupResults.collect{case (modVer, \/-(proj)) => modVer -> proj}
/*
* newProjects are project definitions, fresh from the repositories. We need to add
* dependency management / inheritance-related bits to them.
*/
newProjects.foldLeft(Task.now(copy(errorCache = errors0))) { case (accTask, (modVer, (repo, proj))) =>
for {
current <- accTask
updated <- current.fetch(current.dependencyManagementMissing(proj).toList, fetchModule)
proj0 = updated.withDependencyManagement(proj)
} yield updated.copy(projectCache = updated.projectCache + (modVer -> (repo, proj0)))
}
}
}
def last(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], maxIterations: Int = -1): Task[Resolution] = {
if (maxIterations == 0 || isDone) Task.now(this)
else {
next(fetchModule)
.flatMap(_.last(fetchModule, if (maxIterations > 0) maxIterations - 1 else maxIterations))
}
}
def stream(fetchModule: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)], run: Task[Resolution] => Resolution): Stream[Resolution] = {
this #:: {
if (isDone) Stream.empty
else run(next(fetchModule)).stream(fetchModule, run)
}
}
def minDependencies: Set[Dependency] =
Orders.minDependencies(dependencies)
def artifacts: Seq[Artifact] =
for {
dep <- minDependencies.toSeq
(repo, proj) <- projectCache.get(dep.moduleVersion).toSeq
artifact <- repo.artifacts(dep, proj)
(source, proj) <- projectCache.get(dep.moduleVersion).toSeq
artifact <- source.artifacts(dep, proj)
} yield artifact
def errors: Seq[(Dependency, Seq[String])] =

6
core/src/main/scala/coursier/core/ResolutionProcess.scala
View File

@ -40,9 +40,9 @@ case class Missing(missing: Seq[(Module, String)],
def cont0(res: Resolution) = {
val res0 =
successes.foldLeft(res){case (acc, (modVer, (repo, proj))) =>
successes.foldLeft(res){case (acc, (modVer, (source, proj))) =>
acc.copy(projectCache = acc.projectCache + (
modVer -> (repo, acc.withDependencyManagement(proj))
modVer -> (source, acc.withDependencyManagement(proj))
))
}
Continue(res0, cont)
@ -82,7 +82,7 @@ object ResolutionProcess {
else Missing(resolution0.missingFromCache.toSeq, resolution0, apply)
}
type FetchResult = Seq[((Module, String), Seq[String] \/ (Repository, Project))]
type FetchResult = Seq[((Module, String), Seq[String] \/ (Artifact.Source, Project))]
type Fetch[F[_]] = Seq[(Module, String)] => F[FetchResult]
}

36
core/src/main/scala/coursier/package.scala
View File

@ -1,5 +1,3 @@
import scalaz.{ EitherT, \/ }
import scalaz.concurrent.Task
/**
* Pulls definitions from coursier.core, with default arguments.
@ -65,56 +63,28 @@ package object coursier {
type Scope = core.Scope
val Scope: core.Scope.type = core.Scope
type Repository = core.Repository
def fetchFrom(repositories: Seq[Repository]): ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)] =
modVersion => core.Resolution.find(repositories, modVersion._1, modVersion._2)
def fetchSeveralFrom(repositories: Seq[Repository]): Seq[ModuleVersion] => Task[Seq[(ModuleVersion, Seq[String] \/ (Repository, Project))]] = {
val fetchOne = fetchFrom(repositories)
modVers =>
Task.gatherUnordered(modVers.map(modVer => fetchOne(modVer).run.map(modVer -> _)))
}
type Resolution = core.Resolution
object Resolution {
val empty = apply()
def apply(rootDependencies: Set[Dependency] = Set.empty,
dependencies: Set[Dependency] = Set.empty,
conflicts: Set[Dependency] = Set.empty,
projectCache: Map[ModuleVersion, (Repository, Project)] = Map.empty,
projectCache: Map[ModuleVersion, (Artifact.Source, Project)] = Map.empty,
errorCache: Map[ModuleVersion, Seq[String]] = Map.empty,
filter: Option[Dependency => Boolean] = None,
profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Resolution =
core.Resolution(rootDependencies, dependencies, conflicts, projectCache, errorCache, filter, profileActivation)
}
def resolve(dependencies: Set[Dependency],
fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)],
maxIterations: Option[Int] = Some(200),
filter: Option[Dependency => Boolean] = None,
profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = {
val startResolution = Resolution(
dependencies, Set.empty, Set.empty,
Map.empty, Map.empty,
filter,
profileActivation
)
startResolution.last(fetch, maxIterations.getOrElse(-1))
}
type Artifact = core.Artifact
object Artifact {
def apply(url: String,
extra: Map[String, String] = Map.empty,
attributes: Attributes = Attributes()): Artifact =
core.Artifact(url, extra, attributes)
}
type MavenRepository[G <: core.FetchMetadata] = core.MavenRepository[G]
val MavenRepository: core.MavenRepository.type = core.MavenRepository
type Source = core.Artifact.Source
}
type ResolutionProcess = core.ResolutionProcess
val ResolutionProcess: core.ResolutionProcess.type = core.ResolutionProcess

14
core/src/main/scala/coursier/repository/package.scala
View File

@ -1,14 +0,0 @@
package coursier
import coursier.core.DefaultFetchMetadata
package object repository {
val mavenCentral = MavenRepository(DefaultFetchMetadata("https://repo1.maven.org/maven2/"))
val sonatypeReleases = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/releases/"))
val sonatypeSnapshots = MavenRepository(DefaultFetchMetadata("https://oss.sonatype.org/content/repositories/snapshots/"))
lazy val ivy2Local = MavenRepository(DefaultFetchMetadata("file://" + sys.props("user.home") + "/.ivy2/local/"), ivyLike = true)
}

7
core/src/test/scala/coursier/test/CentralTests.scala
View File

@ -1,6 +1,7 @@
package coursier
package test
import coursier.core.Repository
import utest._
import scala.async.Async.{async, await}
@ -9,12 +10,12 @@ import coursier.test.compatibility._
object CentralTests extends TestSuite {
val repositories = Seq[Repository](
repository.mavenCentral
Repository.mavenCentral
)
def resolve(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = {
ResolutionProcess(Resolution(deps, filter = filter))
.run(fetchSeveralFrom(repositories))
.run(Repository.fetchSeveralFrom(repositories))
.runF
}
@ -83,7 +84,7 @@ object CentralTests extends TestSuite {
assert(res == expected)
assert(res0.projectCache.contains(dep.moduleVersion))
val (_, proj) = res0.projectCache(dep.moduleVersion)
val proj = res0.projectCache(dep.moduleVersion)._2
assert(proj.version == "2.8")
}
}

9
core/src/test/scala/coursier/test/ResolutionTests.scala
View File

@ -1,6 +1,7 @@
package coursier
package test
import coursier.core.Repository
import utest._
import scala.async.Async.{async, await}
@ -10,12 +11,12 @@ object ResolutionTests extends TestSuite {
def resolve0(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = {
ResolutionProcess(Resolution(deps, filter = filter))
.run(fetchSeveralFrom(repositories))
.run(Repository.fetchSeveralFrom(repositories))
.runF
}
implicit class ProjectOps(val p: Project) extends AnyVal {
def kv: (ModuleVersion, (Repository, Project)) = p.moduleVersion -> (testRepository, p)
def kv: (ModuleVersion, (Artifact.Source, Project)) = p.moduleVersion -> (testRepository.source, p)
}
val projects = Seq(
@ -146,7 +147,7 @@ object ResolutionTests extends TestSuite {
)
val projectsMap = projects.map(p => p.moduleVersion -> p).toMap
val testRepository: Repository = new TestRepository(projectsMap)
val testRepository = new TestRepository(projectsMap)
val repositories = Seq[Repository](
testRepository
@ -188,7 +189,7 @@ object ResolutionTests extends TestSuite {
val expected = Resolution(
rootDependencies = Set(dep),
dependencies = Set(dep.withCompileScope),
projectCache = Map(dep.moduleVersion -> (testRepository, projectsMap(dep.moduleVersion)))
projectCache = Map(dep.moduleVersion -> (testRepository.source, projectsMap(dep.moduleVersion)))
)
assert(res == expected)

12
core/src/test/scala/coursier/test/TestRepository.scala
View File

@ -1,16 +1,18 @@
package coursier
package test
import coursier.core.{Versions, CachePolicy}
import coursier.core._
import scalaz.{-\/, \/, EitherT}
import scalaz.EitherT
import scalaz.concurrent.Task
import scalaz.Scalaz._
class TestRepository(projects: Map[(Module, String), Project]) extends Repository {
def find(module: Module, version: String, cachePolicy: CachePolicy) =
val source = new core.Artifact.Source {
def artifacts(dependency: Dependency, project: Project) = ???
}
def find(module: Module, version: String, cachePolicy: Repository.CachePolicy) =
EitherT(Task.now(
projects.get((module, version)).toRightDisjunction("Not found")
projects.get((module, version)).map((source, _)).toRightDisjunction("Not found")
))
def artifacts(dependency: Dependency, project: Project): Seq[Artifact] = ???
}

17
core/src/test/scala/coursier/test/package.scala
View File

@ -1,26 +1,9 @@
package coursier
import scalaz.EitherT
import scalaz.concurrent.Task
package object test {
implicit class DependencyOps(val underlying: Dependency) extends AnyVal {
def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile)
}
def resolve(dependencies: Set[Dependency],
fetch: ModuleVersion => EitherT[Task, Seq[String], (Repository, Project)],
maxIterations: Option[Int] = Some(200),
filter: Option[Dependency => Boolean] = None,
profileActivation: Option[(String, Profile.Activation, Map[String, String]) => Boolean] = None): Task[Resolution] = {
val startResolution = Resolution(
dependencies,
filter = filter,
profileActivation = profileActivation
)
startResolution.last(fetch, maxIterations.getOrElse(-1))
}
}

2
files/src/main/scala/coursier/Files.scala
View File

@ -2,7 +2,7 @@ package coursier
import java.net.{URI, URL}
import coursier.core.CachePolicy
import coursier.core.Repository.CachePolicy
import scala.annotation.tailrec
import scalaz.{-\/, \/-, \/, EitherT}

38
web/src/main/scala/coursier/web/Backend.scala
View File

@ -1,7 +1,7 @@
package coursier
package web
import coursier.core.{DefaultFetchMetadata, Logger}
import coursier.core.{Repository, MavenRepository, Fetch}
import japgolly.scalajs.react.vdom.{TagMod, Attr}
import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml
import japgolly.scalajs.react.{ReactEventI, ReactComponentB, BackendScope}
@ -18,7 +18,7 @@ case class ResolutionOptions(followOptional: Boolean = false,
keepTest: Boolean = false)
case class State(modules: Seq[Dependency],
repositories: Seq[MavenRepository[DefaultFetchMetadata]],
repositories: Seq[MavenRepository],
options: ResolutionOptions,
resolutionOpt: Option[Resolution],
editModuleIdx: Int,
@ -71,7 +71,13 @@ class Backend($: BackendScope[Unit, State]) {
def updateTree(resolution: Resolution, target: String, reverse: Boolean) = {
def depsOf(dep: Dependency) =
resolution.projectCache.get(dep.moduleVersion).toSeq.flatMap(t => core.Resolution.finalDependencies(dep, t._2).filter(resolution.filter getOrElse core.Resolution.defaultFilter))
resolution.projectCache
.get(dep.moduleVersion)
.toSeq
.flatMap{case (_, proj) =>
core.Resolution.finalDependencies(dep, proj)
.filter(resolution.filter getOrElse core.Resolution.defaultFilter)
}
val minDependencies = resolution.minDependencies
@ -105,7 +111,7 @@ class Backend($: BackendScope[Unit, State]) {
g.$("#resLogTab a:last").tab("show")
$.modState(_.copy(resolving = true, log = Nil))
val logger: Logger = new Logger {
val logger: Fetch.Logger = new Fetch.Logger {
def fetched(url: String) = {
println(s"Fetched $url")
$.modState(s => s.copy(log = s"Fetched $url" +: s.log))
@ -128,7 +134,7 @@ class Backend($: BackendScope[Unit, State]) {
)
ResolutionProcess(res)
.run(fetchSeveralFrom(s.repositories.map(r => r.copy(fetchMetadata = r.fetchMetadata.copy(logger = Some(logger))))), 100)
.run(Repository.fetchSeveralFrom(s.repositories.map(r => r.copy(fetch = r.fetch.copy(logger = Some(logger))))), 100)
}
// For reasons that are unclear to me, not delaying this when using the runNow execution context
@ -239,8 +245,8 @@ object App {
)),
<.td(Seq[Seq[TagMod]](
res.projectCache.get(dep.moduleVersion) match {
case Some((MavenRepository(fetchMetadata, _), _)) =>
// FIXME Maven specific, generalize if/when adding support for Ivy
case Some((source: MavenRepository.Source, proj)) if !source.ivyLike =>
// FIXME Maven specific, generalize with source.artifacts
val version0 = finalVersionOpt getOrElse dep.version
val relPath =
dep.module.organization.split('.').toSeq ++ Seq(
@ -249,11 +255,13 @@ object App {
s"${dep.module.name}-$version0"
)
val root = source.root
Seq(
<.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.pom",
<.a(^.href := s"$root${relPath.mkString("/")}.pom",
<.span(^.`class` := "label label-info", "POM")
),
<.a(^.href := s"${fetchMetadata.root}${relPath.mkString("/")}.jar",
<.a(^.href := s"$root${relPath.mkString("/")}.jar",
<.span(^.`class` := "label label-info", "JAR")
)
)
@ -387,19 +395,19 @@ object App {
val modules = dependenciesTable("Dependencies")
val repositories = ReactComponentB[Seq[MavenRepository[DefaultFetchMetadata]]]("Repositories")
val repositories = ReactComponentB[Seq[MavenRepository]]("Repositories")
.render{ repos =>
def repoItem(repo: MavenRepository[DefaultFetchMetadata]) =
def repoItem(repo: MavenRepository) =
<.tr(
<.td(
<.a(^.href := repo.fetchMetadata.root,
repo.fetchMetadata.root
<.a(^.href := repo.fetch.root,
repo.fetch.root
)
)
)
val sortedRepos = repos
.sortBy(repo => repo.fetchMetadata.root)
.sortBy(repo => repo.fetch.root)
<.table(^.`class` := "table",
<.thead(
@ -460,7 +468,7 @@ object App {
}
.build
val initialState = State(Nil, Seq(coursier.repository.mavenCentral), ResolutionOptions(), None, -1, resolving = false, reverseTree = false, log = Nil)
val initialState = State(Nil, Seq(Repository.mavenCentral), ResolutionOptions(), None, -1, resolving = false, reverseTree = false, log = Nil)
val app = ReactComponentB[Unit]("Coursier")
.initialState(initialState)