Merge pull request #92 from alexarchambault/topic/configurations

Basic Ivy support (configurations, repositories), SBT plugin
This commit is contained in:
Alexandre Archambault 2015-12-30 22:51:42 +01:00
commit 903103cf59
54 changed files with 3865 additions and 1621 deletions

View File

@ -51,19 +51,18 @@ lazy val baseCommonSettings = Seq(
organization := "com.github.alexarchambault",
resolvers ++= Seq(
"Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases",
Resolver.sonatypeRepo("releases"),
Resolver.sonatypeRepo("snapshots")
Resolver.sonatypeRepo("releases")
),
scalacOptions += "-target:jvm-1.7",
javacOptions ++= Seq(
"-source", "1.7",
"-target", "1.7"
)
),
javacOptions in Keys.doc := Seq()
)
lazy val commonSettings = baseCommonSettings ++ Seq(
scalaVersion := "2.11.7",
crossScalaVersions := Seq("2.10.6", "2.11.7"),
libraryDependencies ++= {
if (scalaVersion.value startsWith "2.10.")
Seq(compilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full))
@ -72,7 +71,6 @@ lazy val commonSettings = baseCommonSettings ++ Seq(
}
)
lazy val core = crossProject
.settings(commonSettings: _*)
.settings(publishingSettings: _*)
@ -128,17 +126,18 @@ lazy val tests = crossProject
scalaJSStage in Global := FastOptStage
)
lazy val testsJvm = tests.jvm.dependsOn(files % "test")
lazy val testsJvm = tests.jvm.dependsOn(cache % "test")
lazy val testsJs = tests.js.dependsOn(`fetch-js` % "test")
lazy val files = project
lazy val cache = project
.dependsOn(coreJvm)
.settings(commonSettings)
.settings(publishingSettings)
.settings(
name := "coursier-files",
name := "coursier-cache",
libraryDependencies ++= Seq(
"org.scalaz" %% "scalaz-concurrent" % "7.1.2"
"org.scalaz" %% "scalaz-concurrent" % "7.1.2",
"com.lihaoyi" %% "ammonite-terminal" % "0.5.0"
)
)
@ -156,20 +155,19 @@ lazy val bootstrap = project
artifactName0(sv, m, artifact)
},
crossPaths := false,
autoScalaLibrary := false,
javacOptions in doc := Seq()
autoScalaLibrary := false
)
lazy val cli = project
.dependsOn(coreJvm, files)
.dependsOn(coreJvm, cache)
.settings(commonSettings)
.settings(noPublishSettings)
.settings(publishingSettings)
.settings(packAutoSettings)
.settings(
name := "coursier-cli",
libraryDependencies ++= Seq(
"com.github.alexarchambault" %% "case-app" % "1.0.0-SNAPSHOT",
"com.lihaoyi" %% "ammonite-terminal" % "0.5.0",
// beware - available only in 2.11
"com.github.alexarchambault" %% "case-app" % "1.0.0-M1",
"ch.qos.logback" % "logback-classic" % "1.1.3"
),
resourceGenerators in Compile += packageBin.in(bootstrap).in(Compile).map { jar =>
@ -208,8 +206,27 @@ lazy val web = project
)
)
lazy val doc = project
.dependsOn(coreJvm, cache)
.settings(commonSettings)
.settings(noPublishSettings)
.settings(tutSettings)
.settings(
tutSourceDirectory := baseDirectory.value,
tutTargetDirectory := baseDirectory.value / ".."
)
// Don't try to compile that if you're not in 2.10
lazy val plugin = project
.dependsOn(coreJvm, cache)
.settings(baseCommonSettings)
.settings(
name := "coursier-sbt-plugin",
sbtPlugin := true
)
lazy val `coursier` = project.in(file("."))
.aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web)
.aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, cache, bootstrap, cli, web, doc)
.settings(commonSettings)
.settings(noPublishSettings)
.settings(releaseSettings)

View File

@ -0,0 +1,555 @@
package coursier
import java.net.{HttpURLConnection, URL}
import java.nio.channels.{ OverlappingFileLockException, FileLock }
import java.nio.file.{ StandardCopyOption, Files => NioFiles }
import java.security.MessageDigest
import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService}
import scala.annotation.tailrec
import scalaz._
import scalaz.concurrent.{ Task, Strategy }
import java.io.{ Serializable => _, _ }
object Cache {
private def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = {
def local(url: String) =
if (url.startsWith("file:///"))
url.stripPrefix("file://")
else if (url.startsWith("file:/"))
url.stripPrefix("file:")
else {
val localPathOpt = cache.collectFirst {
case (base, cacheDir) if url.startsWith(base) =>
cacheDir + "/" + url.stripPrefix(base)
}
localPathOpt.getOrElse {
// FIXME Means we were handed an artifact from repositories other than the known ones
println(cache.mkString("\n"))
println(url)
???
}
}
if (artifact.extra.contains("local"))
artifact
else
artifact.copy(extra = artifact.extra + ("local" ->
artifact.copy(
url = local(artifact.url),
checksumUrls = artifact.checksumUrls
.mapValues(local)
.toVector
.toMap,
extra = Map.empty
)
))
}
private def readFullyTo(
in: InputStream,
out: OutputStream,
logger: Option[Logger],
url: String,
alreadyDownloaded: Long
): Unit = {
val b = Array.fill[Byte](bufferSize)(0)
@tailrec
def helper(count: Long): Unit = {
val read = in.read(b)
if (read >= 0) {
out.write(b, 0, read)
out.flush()
logger.foreach(_.downloadProgress(url, count + read))
helper(count + read)
}
}
helper(alreadyDownloaded)
}
private def withLockFor[T](file: File)(f: => FileError \/ T): FileError \/ T = {
val lockFile = new File(file.getParentFile, s"${file.getName}.lock")
lockFile.getParentFile.mkdirs()
var out = new FileOutputStream(lockFile)
try {
var lock: FileLock = null
try {
lock = out.getChannel.tryLock()
if (lock == null)
-\/(FileError.Locked(file))
else
try f
finally {
lock.release()
lock = null
out.close()
out = null
lockFile.delete()
}
}
catch {
case e: OverlappingFileLockException =>
-\/(FileError.Locked(file))
}
finally if (lock != null) lock.release()
} finally if (out != null) out.close()
}
private def downloading[T](
url: String,
file: File,
logger: Option[Logger]
)(
f: => FileError \/ T
): FileError \/ T =
try {
val o = new Object
val prev = urlLocks.putIfAbsent(url, o)
if (prev == null) {
logger.foreach(_.downloadingArtifact(url, file))
val res =
try f
catch { case e: Exception =>
logger.foreach(_.downloadedArtifact(url, success = false))
throw e
}
finally {
urlLocks.remove(url)
}
logger.foreach(_.downloadedArtifact(url, success = true))
res
} else
-\/(FileError.ConcurrentDownload(url))
}
catch { case e: Exception =>
-\/(FileError.DownloadError(s"Caught $e (${e.getMessage})"))
}
private def temporaryFile(file: File): File = {
val dir = file.getParentFile
val name = file.getName
new File(dir, s"$name.part")
}
private val partialContentResponseCode = 206
private def download(
artifact: Artifact,
cache: Seq[(String, File)],
checksums: Set[String],
cachePolicy: CachePolicy,
pool: ExecutorService,
logger: Option[Logger] = None
): Task[Seq[((File, String), FileError \/ Unit)]] = {
implicit val pool0 = pool
val artifact0 = withLocal(artifact, cache)
.extra
.getOrElse("local", artifact)
val pairs =
Seq(artifact0.url -> artifact.url) ++ {
checksums
.intersect(artifact0.checksumUrls.keySet)
.intersect(artifact.checksumUrls.keySet)
.toSeq
.map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
}
def urlConn(url: String) = {
val conn = new URL(url).openConnection() // FIXME Should this be closed?
// Dummy user-agent instead of the default "Java/...",
// so that we are not returned incomplete/erroneous metadata
// (Maven 2 compatibility? - happens for snapshot versioning metadata,
// this is SO FSCKING CRAZY)
conn.setRequestProperty("User-Agent", "")
conn
}
def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] =
EitherT {
Task {
\/- {
val lastModified = file.lastModified()
if (lastModified > 0L)
Some(lastModified)
else
None
} : FileError \/ Option[Long]
}
}
def urlLastModified(url: String): EitherT[Task, FileError, Option[Long]] =
EitherT {
Task {
urlConn(url) match {
case c: HttpURLConnection =>
c.setRequestMethod("HEAD")
val remoteLastModified = c.getLastModified
\/- {
if (remoteLastModified > 0L)
Some(remoteLastModified)
else
None
}
case other =>
-\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)"))
}
}
}
def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] =
for {
fileLastModOpt <- fileLastModified(file)
urlLastModOpt <- urlLastModified(url)
} yield {
val fromDatesOpt = for {
fileLastMod <- fileLastModOpt
urlLastMod <- urlLastModOpt
} yield fileLastMod < urlLastMod
fromDatesOpt.getOrElse(true)
}
def remote(file: File, url: String): EitherT[Task, FileError, Unit] =
EitherT {
Task {
withLockFor(file) {
downloading(url, file, logger) {
val tmp = temporaryFile(file)
val alreadyDownloaded = tmp.length()
val conn0 = urlConn(url)
val (partialDownload, conn) = conn0 match {
case conn0: HttpURLConnection if alreadyDownloaded > 0L =>
conn0.setRequestProperty("Range", s"bytes=$alreadyDownloaded-")
if (conn0.getResponseCode == partialContentResponseCode) {
val ackRange = Option(conn0.getHeaderField("Content-Range")).getOrElse("")
if (ackRange.startsWith(s"bytes $alreadyDownloaded-"))
(true, conn0)
else
// unrecognized Content-Range header -> start a new connection with no resume
(false, urlConn(url))
} else
(false, conn0)
case _ => (false, conn0)
}
for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) {
val len = len0 + (if (partialDownload) alreadyDownloaded else 0L)
logger.foreach(_.downloadLength(url, len))
}
val in = new BufferedInputStream(conn.getInputStream, bufferSize)
val result =
try {
tmp.getParentFile.mkdirs()
val out = new FileOutputStream(tmp, partialDownload)
try \/-(readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L))
finally out.close()
} finally in.close()
file.getParentFile.mkdirs()
NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE)
for (lastModified <- Option(conn.getLastModified) if lastModified > 0L)
file.setLastModified(lastModified)
result
}
}
}
}
def checkFileExists(file: File, url: String): EitherT[Task, FileError, Unit] =
EitherT {
Task {
if (file.exists()) {
logger.foreach(_.foundLocally(url, file))
\/-(())
} else
-\/(FileError.NotFound(file.toString))
}
}
val tasks =
for ((f, url) <- pairs) yield {
val file = new File(f)
val res =
if (url.startsWith("file:/")) {
def filtered(s: String) =
s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/")
assert(
filtered(url) == filtered(file.toURI.toString),
s"URL: ${filtered(url)}, file: ${filtered(file.toURI.toString)}"
)
checkFileExists(file, url)
} else
cachePolicy match {
case CachePolicy.LocalOnly =>
checkFileExists(file, url)
case CachePolicy.UpdateChanging | CachePolicy.Update =>
shouldDownload(file, url).flatMap {
case true =>
remote(file, url)
case false =>
EitherT(Task.now(\/-(()) : FileError \/ Unit))
}
case CachePolicy.FetchMissing =>
checkFileExists(file, url) orElse remote(file, url)
case CachePolicy.ForceDownload =>
remote(file, url)
}
res.run.map((file, url) -> _)
}
Nondeterminism[Task].gather(tasks)
}
def validateChecksum(
artifact: Artifact,
sumType: String,
cache: Seq[(String, File)],
pool: ExecutorService
): EitherT[Task, FileError, Unit] = {
implicit val pool0 = pool
val artifact0 = withLocal(artifact, cache)
.extra
.getOrElse("local", artifact)
EitherT {
artifact0.checksumUrls.get(sumType) match {
case Some(sumFile) =>
Task {
val sum = scala.io.Source.fromFile(sumFile)
.getLines()
.toStream
.headOption
.mkString
.takeWhile(!_.isSpaceChar)
val f = new File(artifact0.url)
val md = MessageDigest.getInstance(sumType)
val is = new FileInputStream(f)
val res = try {
var lock: FileLock = null
try {
lock = is.getChannel.tryLock(0L, Long.MaxValue, true)
if (lock == null)
-\/(FileError.Locked(f))
else {
withContent(is, md.update(_, 0, _))
\/-(())
}
}
catch {
case e: OverlappingFileLockException =>
-\/(FileError.Locked(f))
}
finally if (lock != null) lock.release()
} finally is.close()
res.flatMap { _ =>
val digest = md.digest()
val calculatedSum = f"${BigInt(1, digest)}%040x"
if (sum == calculatedSum)
\/-(())
else
-\/(FileError.WrongChecksum(sumType, calculatedSum, sum, artifact0.url, sumFile))
}
}
case None =>
Task.now(-\/(FileError.ChecksumNotFound(sumType, artifact0.url)))
}
}
}
def file(
artifact: Artifact,
cache: Seq[(String, File)],
cachePolicy: CachePolicy,
checksums: Seq[Option[String]] = Seq(Some("SHA-1")),
logger: Option[Logger] = None,
pool: ExecutorService = defaultPool
): EitherT[Task, FileError, File] = {
implicit val pool0 = pool
val checksums0 = if (checksums.isEmpty) Seq(None) else checksums
val res = EitherT {
download(
artifact,
cache,
checksums = checksums0.collect { case Some(c) => c }.toSet,
cachePolicy,
pool,
logger = logger
).map { results =>
val checksum = checksums0.find {
case None => true
case Some(c) =>
artifact.checksumUrls.get(c).exists { cUrl =>
results.exists { case ((_, u), b) =>
u == cUrl && b.isRight
}
}
}
val ((f, _), res) = results.head
res.flatMap { _ =>
checksum match {
case None =>
// FIXME All the checksums should be in the error, possibly with their URLs
// from artifact.checksumUrls
-\/(FileError.ChecksumNotFound(checksums0.last.get, ""))
case Some(c) => \/-((f, c))
}
}
}
}
res.flatMap {
case (f, None) => EitherT(Task.now[FileError \/ File](\/-(f)))
case (f, Some(c)) =>
validateChecksum(artifact, c, cache, pool).map(_ => f)
}
}
def fetch(
cache: Seq[(String, File)],
cachePolicy: CachePolicy,
checksums: Seq[Option[String]] = Seq(Some("SHA-1")),
logger: Option[Logger] = None,
pool: ExecutorService = defaultPool
): Fetch.Content[Task] = {
artifact =>
file(
artifact,
cache,
cachePolicy,
checksums = checksums,
logger = logger,
pool = pool
).leftMap(_.message).map { f =>
// FIXME Catch error here?
scala.io.Source.fromFile(f)("UTF-8").mkString
}
}
lazy val ivy2Local = MavenRepository(
new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString,
ivyLike = true
)
val defaultConcurrentDownloadCount = 6
lazy val defaultPool =
Executors.newFixedThreadPool(defaultConcurrentDownloadCount, Strategy.DefaultDaemonThreadFactory)
private val urlLocks = new ConcurrentHashMap[String, Object]
trait Logger {
def foundLocally(url: String, f: File): Unit = {}
def downloadingArtifact(url: String, file: File): Unit = {}
def downloadLength(url: String, length: Long): Unit = {}
def downloadProgress(url: String, downloaded: Long): Unit = {}
def downloadedArtifact(url: String, success: Boolean): Unit = {}
}
var bufferSize = 1024*1024
def readFullySync(is: InputStream) = {
val buffer = new ByteArrayOutputStream()
val data = Array.ofDim[Byte](16384)
var nRead = is.read(data, 0, data.length)
while (nRead != -1) {
buffer.write(data, 0, nRead)
nRead = is.read(data, 0, data.length)
}
buffer.flush()
buffer.toByteArray
}
def readFully(is: => InputStream) =
Task {
\/.fromTryCatchNonFatal {
val is0 = is
val b =
try readFullySync(is0)
finally is0.close()
new String(b, "UTF-8")
} .leftMap(_.getMessage)
}
def withContent(is: InputStream, f: (Array[Byte], Int) => Unit): Unit = {
val data = Array.ofDim[Byte](16384)
var nRead = is.read(data, 0, data.length)
while (nRead != -1) {
f(data, nRead)
nRead = is.read(data, 0, data.length)
}
}
}
sealed trait FileError extends Product with Serializable {
def message: String
}
object FileError {
case class DownloadError(message0: String) extends FileError {
def message = s"Download error: $message0"
}
case class NotFound(file: String) extends FileError {
def message = s"$file: not found"
}
case class ChecksumNotFound(sumType: String, file: String) extends FileError {
def message = s"$file: $sumType checksum not found"
}
case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError {
def message = s"$file: $sumType checksum validation failed"
}
sealed trait Recoverable extends FileError
case class Locked(file: File) extends Recoverable {
def message = s"$file: locked"
}
case class ConcurrentDownload(url: String) extends Recoverable {
def message = s"$url: concurrent download"
}
}

View File

@ -0,0 +1,11 @@
package coursier
sealed trait CachePolicy extends Product with Serializable
object CachePolicy {
case object LocalOnly extends CachePolicy
case object UpdateChanging extends CachePolicy
case object Update extends CachePolicy
case object FetchMissing extends CachePolicy
case object ForceDownload extends CachePolicy
}

View File

@ -39,7 +39,7 @@ object Platform {
}
}
val artifact: Repository.Fetch[Task] = { artifact =>
val artifact: Fetch.Content[Task] = { artifact =>
EitherT {
val url = new URL(artifact.url)
@ -53,4 +53,9 @@ object Platform {
}
}
implicit def fetch(
repositories: Seq[core.Repository]
): Fetch.Metadata[Task] =
Fetch(repositories, Platform.artifact)
}

View File

@ -0,0 +1,234 @@
package coursier
import java.io.{File, Writer}
import java.util.concurrent._
import ammonite.terminal.{ TTY, Ansi }
import scala.annotation.tailrec
import scala.collection.mutable.ArrayBuffer
class TermDisplay(
out: Writer,
var fallbackMode: Boolean = false
) extends Cache.Logger {
private val ansi = new Ansi(out)
private var width = 80
private val refreshInterval = 1000 / 60
private val fallbackRefreshInterval = 1000
private val lock = new AnyRef
private var currentHeight = 0
private val t = new Thread("TermDisplay") {
override def run() = lock.synchronized {
val baseExtraWidth = width / 5
def reflowed(url: String, info: Info) = {
val pctOpt = info.pct.map(100.0 * _)
val extra =
if (info.length.isEmpty && info.downloaded == 0L)
""
else
s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${info.downloaded}${info.length.map(" / " + _).mkString})"
val total = url.length + 1 + extra.length
val (url0, extra0) =
if (total >= width) { // or > ? If equal, does it go down 2 lines?
val overflow = total - width + 1
val extra0 =
if (extra.length > baseExtraWidth)
extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…"
else
extra
val total0 = url.length + 1 + extra0.length
val overflow0 = total0 - width + 1
val url0 =
if (total0 >= width)
url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…"
else
url
(url0, extra0)
} else
(url, extra)
(url0, extra0)
}
@tailrec def helper(lineCount: Int): Unit = {
currentHeight = lineCount
Option(q.poll(100L, TimeUnit.MILLISECONDS)) match {
case None => helper(lineCount)
case Some(Left(())) => // poison pill
case Some(Right(())) =>
// update display
val downloads0 = downloads.synchronized {
downloads
.toVector
.map { url => url -> infos.get(url) }
.sortBy { case (_, info) => - info.pct.sum }
}
for ((url, info) <- downloads0) {
assert(info != null, s"Incoherent state ($url)")
val (url0, extra0) = reflowed(url, info)
ansi.clearLine(2)
out.write(s"$url0 $extra0\n")
}
if (downloads0.length < lineCount) {
for (_ <- downloads0.length until lineCount) {
ansi.clearLine(2)
ansi.down(1)
}
for (_ <- downloads0.length until lineCount)
ansi.up(1)
}
for (_ <- downloads0.indices)
ansi.up(1)
out.flush()
Thread.sleep(refreshInterval)
helper(downloads0.length)
}
}
@tailrec def fallbackHelper(previous: Set[String]): Unit =
Option(q.poll(100L, TimeUnit.MILLISECONDS)) match {
case None => fallbackHelper(previous)
case Some(Left(())) => // poison pill
case Some(Right(())) =>
val downloads0 = downloads.synchronized {
downloads
.toVector
.map { url => url -> infos.get(url) }
.sortBy { case (_, info) => - info.pct.sum }
}
var displayedSomething = false
for ((url, info) <- downloads0 if previous(url)) {
assert(info != null, s"Incoherent state ($url)")
val (url0, extra0) = reflowed(url, info)
displayedSomething = true
out.write(s"$url0 $extra0\n")
}
if (displayedSomething)
out.write("\n")
out.flush()
Thread.sleep(fallbackRefreshInterval)
fallbackHelper(previous ++ downloads0.map { case (url, _) => url })
}
if (fallbackMode)
fallbackHelper(Set.empty)
else
helper(0)
}
}
t.setDaemon(true)
def init(): Unit = {
try {
width = TTY.consoleDim("cols")
ansi.clearLine(2)
} catch { case _: Exception =>
fallbackMode = true
}
t.start()
}
def stop(): Unit = {
for (_ <- 0 until currentHeight) {
ansi.clearLine(2)
ansi.down(1)
}
for (_ <- 0 until currentHeight) {
ansi.up(1)
}
q.put(Left(()))
lock.synchronized(())
}
private case class Info(downloaded: Long, length: Option[Long]) {
def pct: Option[Double] = length.map(downloaded.toDouble / _)
}
private val downloads = new ArrayBuffer[String]
private val infos = new ConcurrentHashMap[String, Info]
private val q = new LinkedBlockingDeque[Either[Unit, Unit]]
def update(): Unit = {
if (q.size() == 0)
q.put(Right(()))
}
override def downloadingArtifact(url: String, file: File): Unit = {
assert(!infos.containsKey(url))
val prev = infos.putIfAbsent(url, Info(0L, None))
assert(prev == null)
if (fallbackMode) {
// FIXME What about concurrent accesses to out from the thread above?
out.write(s"Downloading $url\n")
out.flush()
}
downloads.synchronized {
downloads.append(url)
}
update()
}
override def downloadLength(url: String, length: Long): Unit = {
val info = infos.get(url)
assert(info != null)
val newInfo = info.copy(length = Some(length))
infos.put(url, newInfo)
update()
}
override def downloadProgress(url: String, downloaded: Long): Unit = {
val info = infos.get(url)
assert(info != null)
val newInfo = info.copy(downloaded = downloaded)
infos.put(url, newInfo)
update()
}
override def downloadedArtifact(url: String, success: Boolean): Unit = {
downloads.synchronized {
downloads -= url
}
if (fallbackMode) {
// FIXME What about concurrent accesses to out from the thread above?
out.write(s"Downloaded $url\n")
out.flush()
}
val info = infos.remove(url)
assert(info != null)
update()
}
}

View File

@ -12,12 +12,10 @@ import coursier.util.ClasspathFilter
case class CommonOptions(
@HelpMessage("Keep optional dependencies (Maven)")
keepOptional: Boolean,
@HelpMessage("Off-line mode: only use cache and local repositories")
@ExtraName("c")
offline: Boolean,
@HelpMessage("Force download: for remote repositories only: re-download items, that is, don't use cache directly")
@ExtraName("f")
force: Boolean,
@HelpMessage("Download mode (default: missing, that is fetch things missing from cache)")
@ValueDescription("offline|update-changing|update|missing|force")
@ExtraName("m")
mode: String = "missing",
@HelpMessage("Quiet output")
@ExtraName("q")
quiet: Boolean,
@ -30,6 +28,10 @@ case class CommonOptions(
@HelpMessage("Repositories - for multiple repositories, separate with comma and/or repeat this option (e.g. -r central,ivy2local -r sonatype-snapshots, or equivalently -r central,ivy2local,sonatype-snapshots)")
@ExtraName("r")
repository: List[String],
@HelpMessage("Do not add default repositories (~/.ivy2/local, and Central)")
noDefault: Boolean = false,
@HelpMessage("Modify names in Maven repository paths for SBT plugins")
sbtPluginHack: Boolean = false,
@HelpMessage("Force module version")
@ValueDescription("organization:name:forcedVersion")
@ExtraName("V")
@ -66,19 +68,28 @@ case class Fetch(
@HelpMessage("Fetch javadoc artifacts")
@ExtraName("D")
javadoc: Boolean,
@HelpMessage("Print java -cp compatible output")
@ExtraName("p")
classpath: Boolean,
@Recurse
common: CommonOptions
) extends CoursierCommand {
val helper = new Helper(common, remainingArgs)
val files0 = helper.fetch(main = true, sources = false, javadoc = false)
val files0 = helper.fetch(sources = sources, javadoc = javadoc)
println(
files0
.map(_.toString)
.mkString("\n")
)
val out =
if (classpath)
files0
.map(_.toString)
.mkString(File.pathSeparator)
else
files0
.map(_.toString)
.mkString("\n")
println(out)
}
@ -86,6 +97,9 @@ case class Launch(
@ExtraName("M")
@ExtraName("main")
mainClass: String,
@ExtraName("c")
@HelpMessage("Assume coursier is a dependency of the launched app, and share the coursier dependency of the launcher with it - allows the launched app to get the resolution that launched it via ResolutionClassLoader")
addCoursier: Boolean,
@Recurse
common: CommonOptions
) extends CoursierCommand {
@ -99,15 +113,37 @@ case class Launch(
}
}
val helper = new Helper(common, rawDependencies)
val extraForceVersions =
if (addCoursier)
???
else
Seq.empty[String]
val files0 = helper.fetch(main = true, sources = false, javadoc = false)
val dontFilterOut =
if (addCoursier) {
val url = classOf[coursier.core.Resolution].getProtectionDomain.getCodeSource.getLocation
if (url.getProtocol == "file")
Seq(new File(url.getPath))
else {
Console.err.println(s"Cannot get the location of the JAR of coursier ($url not a file URL)")
sys.exit(255)
}
} else
Seq.empty[File]
val helper = new Helper(
common.copy(forceVersion = common.forceVersion ++ extraForceVersions),
rawDependencies
)
val files0 = helper.fetch(sources = false, javadoc = false)
val cl = new URLClassLoader(
files0.map(_.toURI.toURL).toArray,
new ClasspathFilter(
Thread.currentThread().getContextClassLoader,
Coursier.baseCp.map(new File(_)).toSet,
Coursier.baseCp.map(new File(_)).toSet -- dontFilterOut,
exclude = true
)
)
@ -169,108 +205,6 @@ case class Launch(
method.invoke(null, extraArgs.toArray)
}
case class Classpath(
@Recurse
common: CommonOptions
) extends CoursierCommand {
val helper = new Helper(common, remainingArgs)
val files0 = helper.fetch(main = true, sources = false, javadoc = false)
Console.out.println(
files0
.map(_.toString)
.mkString(File.pathSeparator)
)
}
// TODO: allow removing a repository (with confirmations, etc.)
case class Repository(
@ValueDescription("id:baseUrl")
@ExtraName("a")
add: List[String],
@ExtraName("L")
list: Boolean,
@ExtraName("l")
defaultList: Boolean,
ivyLike: Boolean,
@Recurse
cacheOptions: CacheOptions
) extends CoursierCommand {
if (add.exists(!_.contains(":"))) {
CaseApp.printUsage[Repository](err = true)
sys.exit(255)
}
val add0 = add
.map{ s =>
val Seq(id, baseUrl) = s.split(":", 2).toSeq
id -> baseUrl
}
if (
add0.exists(_._1.contains("/")) ||
add0.exists(_._1.startsWith(".")) ||
add0.exists(_._1.isEmpty)
) {
CaseApp.printUsage[Repository](err = true)
sys.exit(255)
}
val cache = Cache(new File(cacheOptions.cache))
if (cache.cache.exists() && !cache.cache.isDirectory) {
Console.err.println(s"Error: ${cache.cache} not a directory")
sys.exit(1)
}
if (!cache.cache.exists())
cache.init(verbose = true)
val current = cache.list().map(_._1).toSet
val alreadyAdded = add0
.map(_._1)
.filter(current)
if (alreadyAdded.nonEmpty) {
Console.err.println(s"Error: already added: ${alreadyAdded.mkString(", ")}")
sys.exit(1)
}
for ((id, baseUrl0) <- add0) {
val baseUrl =
if (baseUrl0.endsWith("/"))
baseUrl0
else
baseUrl0 + "/"
cache.add(id, baseUrl, ivyLike = ivyLike)
}
if (defaultList) {
val map = cache.repositoryMap()
for (id <- cache.default(withNotFound = true))
map.get(id) match {
case Some(repo) =>
println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else ""))
case None =>
println(s"$id (not found)")
}
}
if (list)
for ((id, repo, _) <- cache.list().sortBy(_._1)) {
println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else ""))
}
}
case class Bootstrap(
@ExtraName("M")
@ExtraName("main")
@ -325,7 +259,7 @@ case class Bootstrap(
val bootstrapJar =
Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match {
case Some(is) => Files.readFullySync(is)
case Some(is) => Cache.readFullySync(is)
case None =>
Console.err.println(s"Error: bootstrap JAR not found")
sys.exit(1)

View File

@ -2,28 +2,14 @@ package coursier
package cli
import java.io.{ OutputStreamWriter, File }
import java.util.UUID
import java.util.concurrent.Executors
import coursier.ivy.IvyRepository
import scalaz.{ \/-, -\/ }
import scalaz.concurrent.Task
import scalaz.concurrent.{ Task, Strategy }
object Helper {
def validate(common: CommonOptions) = {
import common._
if (force && offline) {
Console.err.println("Error: --offline (-c) and --force (-f) options can't be specified at the same time.")
sys.exit(255)
}
if (parallel <= 0) {
Console.err.println(s"Error: invalid --parallel (-n) value: $parallel")
sys.exit(255)
}
???
}
def fileRepr(f: File) = f.toString
def errPrintln(s: String) = Console.err.println(s)
@ -54,77 +40,91 @@ class Helper(
import common._
import Helper.errPrintln
implicit val cachePolicy =
if (offline)
CachePolicy.LocalOnly
else if (force)
CachePolicy.ForceDownload
else
CachePolicy.Default
val cache = Cache(new File(cacheOptions.cache))
cache.init(verbose = verbose0 >= 0)
val repositoryIds = {
val repositoryIds0 = repository
.flatMap(_.split(','))
.map(_.trim)
.filter(_.nonEmpty)
if (repositoryIds0.isEmpty)
cache.default()
else
repositoryIds0
val cachePolicies = mode match {
case "offline" =>
Seq(CachePolicy.LocalOnly)
case "update-changing" =>
Seq(CachePolicy.UpdateChanging)
case "update" =>
Seq(CachePolicy.Update)
case "missing" =>
Seq(CachePolicy.FetchMissing)
case "force" =>
Seq(CachePolicy.ForceDownload)
case "default" =>
Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing)
case other =>
errPrintln(s"Unrecognized mode: $other")
sys.exit(255)
}
val repoMap = cache.map()
val repoByBase = repoMap.map { case (_, v @ (m, _)) =>
m.root -> v
}
val caches =
Seq(
"http://" -> new File(new File(cacheOptions.cache), "http"),
"https://" -> new File(new File(cacheOptions.cache), "https")
)
val repositoryIdsOpt0 = repositoryIds.map { id =>
repoMap.get(id) match {
case Some(v) => Right(v)
case None =>
if (id.contains("://")) {
val root0 = if (id.endsWith("/")) id else id + "/"
Right(
repoByBase.getOrElse(root0, {
val id0 = UUID.randomUUID().toString
if (verbose0 >= 1)
Console.err.println(s"Addding repository $id0 ($root0)")
val pool = Executors.newFixedThreadPool(parallel, Strategy.DefaultDaemonThreadFactory)
// FIXME This could be done more cleanly
cache.add(id0, root0, ivyLike = false)
cache.map().getOrElse(id0,
sys.error(s"Adding repository $id0 ($root0)")
)
})
)
} else
Left(id)
val central = MavenRepository("https://repo1.maven.org/maven2/")
val ivy2Local = MavenRepository(
new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString,
ivyLike = true
)
val defaultRepositories = Seq(
ivy2Local,
central
)
val repositories0 = common.repository.map { repo =>
val repo0 = repo.toLowerCase
if (repo0 == "central")
Right(central)
else if (repo0 == "ivy2local")
Right(ivy2Local)
else if (repo0.startsWith("sonatype:"))
Right(
MavenRepository(s"https://oss.sonatype.org/content/repositories/${repo.drop("sonatype:".length)}")
)
else {
val (url, r) =
if (repo.startsWith("ivy:")) {
val url = repo.drop("ivy:".length)
(url, IvyRepository(url))
} else if (repo.startsWith("ivy-like:")) {
val url = repo.drop("ivy-like:".length)
(url, MavenRepository(url, ivyLike = true))
} else {
(repo, MavenRepository(repo))
}
if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/"))
Right(r)
else
Left(repo -> s"Unrecognized protocol or repository: $url")
}
}
val notFoundRepositoryIds = repositoryIdsOpt0.collect {
case Left(id) => id
}
if (notFoundRepositoryIds.nonEmpty) {
errPrintln(
(if (notFoundRepositoryIds.lengthCompare(1) == 0) "Repository" else "Repositories") +
" not found: " +
notFoundRepositoryIds.mkString(", ")
)
val unrecognizedRepos = repositories0.collect { case Left(e) => e }
if (unrecognizedRepos.nonEmpty) {
errPrintln(s"${unrecognizedRepos.length} error(s) parsing repositories:")
for ((repo, err) <- unrecognizedRepos)
errPrintln(s"$repo: $err")
sys.exit(255)
}
val files = cache.files().copy(concurrentDownloadCount = parallel)
val repositories1 =
(if (common.noDefault) Nil else defaultRepositories) ++
repositories0.collect { case Right(r) => r }
val (repositories, fileCaches) = repositoryIdsOpt0
.collect { case Right(v) => v }
.unzip
val repositories =
if (common.sbtPluginHack)
repositories1.map {
case m: MavenRepository => m.copy(sbtAttrStub = true)
case other => other
}
else
repositories1
val (rawDependencies, extraArgs) = {
val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0)
@ -166,12 +166,25 @@ class Helper(
}
val moduleVersions = splitDependencies.map{
case Seq(org, name, version) =>
(Module(org, name), version)
case Seq(org, namePart, version) =>
val p = namePart.split(';')
val name = p.head
val splitAttributes = p.tail.map(_.split("=", 2).toSeq).toSeq
val malformedAttributes = splitAttributes.filter(_.length != 2)
if (malformedAttributes.nonEmpty) {
// FIXME Get these for all dependencies at once
Console.err.println(s"Malformed attributes in ${splitDependencies.mkString(":")}")
// :(
sys.exit(255)
}
val attributes = splitAttributes.collect {
case Seq(k, v) => k -> v
}
(Module(org, name, attributes.toMap), version)
}
val deps = moduleVersions.map{case (mod, ver) =>
Dependency(mod, ver, scope = Scope.Runtime)
Dependency(mod, ver, configuration = "runtime")
}
val forceVersions = {
@ -200,11 +213,14 @@ class Helper(
Some(new TermDisplay(new OutputStreamWriter(System.err)))
else
None
logger.foreach(_.init())
val fetchs = cachePolicies.map(p =>
Cache.fetch(caches, p, logger = logger, pool = pool)
)
val fetchQuiet = coursier.Fetch(
repositories,
files.fetch(logger = logger)(cachePolicy = CachePolicy.LocalOnly), // local files get the priority
files.fetch(logger = logger)
fetchs.head,
fetchs.tail: _*
)
val fetch0 =
if (verbose0 <= 0) fetchQuiet
@ -229,6 +245,7 @@ class Helper(
}
}
logger.foreach(_.init())
val res = startRes
.process
@ -296,35 +313,47 @@ class Helper(
}
}
def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = {
if (verbose0 >= 0)
errPrintln("Fetching artifacts")
val artifacts0 = res.artifacts
val main0 = main || (!sources && !javadoc)
val artifacts = artifacts0.flatMap{ artifact =>
var l = List.empty[Artifact]
if (sources)
l = artifact.extra.get("sources").toList ::: l
if (javadoc)
l = artifact.extra.get("javadoc").toList ::: l
if (main0)
l = artifact :: l
def fetch(sources: Boolean, javadoc: Boolean): Seq[File] = {
if (verbose0 >= 0) {
val msg = cachePolicies match {
case Seq(CachePolicy.LocalOnly) =>
"Checking artifacts"
case _ =>
"Fetching artifacts"
}
l
errPrintln(msg)
}
val artifacts =
if (sources || javadoc) {
var classifiers = Seq.empty[String]
if (sources)
classifiers = classifiers :+ "sources"
if (javadoc)
classifiers = classifiers :+ "javadoc"
res.classifiersArtifacts(classifiers)
} else
res.artifacts
val logger =
if (verbose0 >= 0)
Some(new TermDisplay(new OutputStreamWriter(System.err)))
else
None
if (verbose0 >= 1 && artifacts.nonEmpty)
println(s"Found ${artifacts.length} artifacts")
val tasks = artifacts.map(artifact =>
(Cache.file(artifact, caches, cachePolicies.head, logger = logger, pool = pool) /: cachePolicies.tail)(
_ orElse Cache.file(artifact, caches, _, logger = logger, pool = pool)
).run.map(artifact.->)
)
logger.foreach(_.init())
val tasks = artifacts.map(artifact => files.file(artifact, logger = logger).run.map(artifact.->))
def printTask = Task {
if (verbose0 >= 1 && artifacts.nonEmpty)
println(s"Found ${artifacts.length} artifacts")
}
val task = printTask.flatMap(_ => Task.gatherUnordered(tasks))
val task = Task.gatherUnordered(tasks)
val results = task.run
val errors = results.collect{case (artifact, -\/(err)) => artifact -> err }

View File

@ -1,150 +0,0 @@
package coursier.cli
import java.io.Writer
import java.util.concurrent._
import ammonite.terminal.{ TTY, Ansi }
import coursier.Files.Logger
import scala.annotation.tailrec
import scala.collection.mutable.ArrayBuffer
class TermDisplay(out: Writer) extends Logger {
private val ansi = new Ansi(out)
private var width = 80
private val refreshInterval = 1000 / 60
private val lock = new AnyRef
private val t = new Thread("TermDisplay") {
override def run() = lock.synchronized {
val baseExtraWidth = width / 5
@tailrec def helper(lineCount: Int): Unit =
Option(q.poll(100L, TimeUnit.MILLISECONDS)) match {
case None => helper(lineCount)
case Some(Left(())) => // poison pill
case Some(Right(())) =>
// update display
for (_ <- 0 until lineCount) {
ansi.up(1)
ansi.clearLine(2)
}
val downloads0 = downloads.synchronized {
downloads
.toVector
.map { url => url -> infos.get(url) }
.sortBy { case (_, info) => - info.pct.sum }
}
for ((url, info) <- downloads0) {
assert(info != null, s"Incoherent state ($url)")
val pctOpt = info.pct.map(100.0 * _)
val extra =
if (info.length.isEmpty && info.downloaded == 0L)
""
else
s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${info.downloaded}${info.length.map(" / " + _).mkString})"
val total = url.length + 1 + extra.length
val (url0, extra0) =
if (total >= width) { // or > ? If equal, does it go down 2 lines?
val overflow = total - width + 1
val extra0 =
if (extra.length > baseExtraWidth)
extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…"
else
extra
val total0 = url.length + 1 + extra0.length
val overflow0 = total0 - width + 1
val url0 =
if (total0 >= width)
url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…"
else
url
(url0, extra0)
} else
(url, extra)
out.write(s"$url0 $extra0\n")
}
out.flush()
Thread.sleep(refreshInterval)
helper(downloads0.length)
}
helper(0)
}
}
t.setDaemon(true)
def init(): Unit = {
width = TTY.consoleDim("cols")
ansi.clearLine(2)
t.start()
}
def stop(): Unit = {
q.put(Left(()))
lock.synchronized(())
}
private case class Info(downloaded: Long, length: Option[Long]) {
def pct: Option[Double] = length.map(downloaded.toDouble / _)
}
private val downloads = new ArrayBuffer[String]
private val infos = new ConcurrentHashMap[String, Info]
private val q = new LinkedBlockingDeque[Either[Unit, Unit]]
def update(): Unit = {
if (q.size() == 0)
q.put(Right(()))
}
override def downloadingArtifact(url: String): Unit = {
assert(!infos.containsKey(url))
val prev = infos.putIfAbsent(url, Info(0L, None))
assert(prev == null)
downloads.synchronized {
downloads.append(url)
}
update()
}
override def downloadLength(url: String, length: Long): Unit = {
val info = infos.get(url)
assert(info != null)
val newInfo = info.copy(length = Some(length))
infos.put(url, newInfo)
update()
}
override def downloadProgress(url: String, downloaded: Long): Unit = {
val info = infos.get(url)
assert(info != null)
val newInfo = info.copy(downloaded = downloaded)
infos.put(url, newInfo)
update()
}
override def downloadedArtifact(url: String, success: Boolean): Unit = {
downloads.synchronized {
downloads -= url
}
val info = infos.remove(url)
assert(info != null)
update()
}
}

View File

@ -46,11 +46,13 @@ package object compatibility {
def label =
option[String](node0.nodeName)
.getOrElse("")
def child =
def children =
option[NodeList](node0.childNodes)
.map(l => List.tabulate(l.length)(l.item).map(fromNode))
.getOrElse(Nil)
def attributes = ???
// `exists` instead of `contains`, for scala 2.10
def isText =
option[Int](node0.nodeType)

View File

@ -0,0 +1,37 @@
package coursier
import java.io.File
import java.net.URLClassLoader
import coursier.util.ClasspathFilter
class ResolutionClassLoader(
val resolution: Resolution,
val artifacts: Seq[(Dependency, Artifact, File)],
parent: ClassLoader
) extends URLClassLoader(
artifacts.map { case (_, _, f) => f.toURI.toURL }.toArray,
parent
) {
/**
* Filtered version of this `ClassLoader`, exposing only `dependencies` and their
* their transitive dependencies, and filtering out the other dependencies from
* `resolution` - for `ClassLoader` isolation.
*
* An application launched by `coursier launch -C` has `ResolutionClassLoader` set as its
* context `ClassLoader` (can be obtain with `Thread.currentThread().getContextClassLoader`).
* If it aims at doing `ClassLoader` isolation, exposing only a dependency `dep` to the isolated
* things, `filter(dep)` provides a `ClassLoader` that loaded `dep` and all its transitive
* dependencies through the same loader as the contextual one, but that "exposes" only
* `dep` and its transitive dependencies, nothing more.
*/
def filter(dependencies: Set[Dependency]): ClassLoader = {
val subRes = resolution.subset(dependencies)
val subArtifacts = subRes.dependencyArtifacts.map { case (_, a) => a }.toSet
val subFiles = artifacts.collect { case (_, a, f) if subArtifacts(a) => f }
new ClasspathFilter(this, subFiles.toSet, exclude = false)
}
}

View File

@ -2,6 +2,8 @@ package coursier.core
import coursier.util.Xml
import scala.xml.{ Attribute, MetaData, Null }
package object compatibility {
implicit class RichChar(val c: Char) extends AnyVal {
@ -16,11 +18,32 @@ package object compatibility {
def fromNode(node: scala.xml.Node): Xml.Node =
new Xml.Node {
lazy val attributes = {
def helper(m: MetaData): Stream[(String, String, String)] =
m match {
case Null => Stream.empty
case attr =>
val pre = attr match {
case a: Attribute => Option(node.getNamespace(a.pre)).getOrElse("")
case _ => ""
}
val value = attr.value.collect {
case scala.xml.Text(t) => t
}.mkString("")
(pre, attr.key, value) #:: helper(m.next)
}
helper(node.attributes).toVector
}
def label = node.label
def child = node.child.map(fromNode)
def children = node.child.map(fromNode)
def isText = node match { case _: scala.xml.Text => true; case _ => false }
def textContent = node.text
def isElement = node match { case _: scala.xml.Elem => true; case _ => false }
override def toString = node.toString
}
parse.right

View File

@ -84,9 +84,23 @@ class ClasspathFilter(parent: ClassLoader, classpath: Set[File], exclude: Boolea
override def loadClass(className: String, resolve: Boolean): Class[_] = {
val c = super.loadClass(className, resolve)
if (fromClasspath(c)) c
else throw new ClassNotFoundException(className)
val c =
try super.loadClass(className, resolve)
catch {
case e: LinkageError =>
// Happens when trying to derive a shapeless.Generic
// from an Ammonite session launched like
// ./coursier launch com.lihaoyi:ammonite-repl_2.11.7:0.5.2
// For className == "shapeless.GenericMacros",
// the super.loadClass above - which would be filtered out below anyway,
// raises a NoClassDefFoundError.
null
}
if (c != null && fromClasspath(c))
c
else
throw new ClassNotFoundException(className)
}
override def getResource(name: String): URL = {

View File

@ -0,0 +1,104 @@
package coursier
import coursier.maven.MavenSource
import scalaz._
object Fetch {
type Content[F[_]] = Artifact => EitherT[F, String, String]
type MD = Seq[(
(Module, String),
Seq[String] \/ (Artifact.Source, Project)
)]
type Metadata[F[_]] = Seq[(Module, String)] => F[MD]
/**
* Try to find `module` among `repositories`.
*
* Look at `repositories` from the left, one-by-one, and stop at first success.
* Else, return all errors, in the same order.
*
* The `version` field of the returned `Project` in case of success may not be
* equal to the provided one, in case the latter is not a specific
* version (e.g. version interval). Which version get chosen depends on
* the repository implementation.
*/
def find[F[_]](
repositories: Seq[Repository],
module: Module,
version: String,
fetch: Content[F]
)(implicit
F: Monad[F]
): EitherT[F, Seq[String], (Artifact.Source, Project)] = {
val lookups = repositories
.map(repo => repo -> repo.find(module, version, fetch).run)
val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) {
case (acc, (repo, eitherProjTask)) =>
val looseModuleValidation = repo match {
case m: MavenRepository => m.sbtAttrStub // that sucks so much
case _ => false
}
val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module
F.bind(acc) {
case -\/(errors) =>
F.map(eitherProjTask)(_.flatMap{case (source, project) =>
val projModule =
if (looseModuleValidation)
project.module.copy(attributes = Map.empty)
else
project.module
if (projModule == moduleCmp) \/-((source, project))
else -\/(s"Wrong module returned (expected: $moduleCmp, got: ${project.module})")
}.leftMap(error => error +: errors))
case res @ \/-(_) =>
F.point(res)
}
}
EitherT(F.map(task)(_.leftMap(_.reverse)))
.map {case x @ (source, proj) =>
val looseModuleValidation = source match {
case m: MavenSource => m.sbtAttrStub // omfg
case _ => false
}
val projModule =
if (looseModuleValidation)
proj.module.copy(attributes = Map.empty)
else
proj.module
val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module
assert(projModule == moduleCmp)
x
}
}
def apply[F[_]](
repositories: Seq[core.Repository],
fetch: Content[F],
extra: Content[F]*
)(implicit
F: Nondeterminism[F]
): Metadata[F] = {
modVers =>
F.map(
F.gatherUnordered(
modVers.map { case (module, version) =>
def get(fetch: Content[F]) =
find(repositories, module, version, fetch)
F.map((get(fetch) /: extra)(_ orElse get(_))
.run)((module, version) -> _)
}
)
)(_.toSeq)
}
}

View File

@ -8,12 +8,11 @@ package coursier.core
* between them.
*
* Using the same terminology as Ivy.
*
* Ivy attributes would land here, if support for it is added.
*/
case class Module(
organization: String,
name: String
name: String,
attributes: Map[String, String]
) {
def trim: Module = copy(
@ -21,10 +20,15 @@ case class Module(
name = name.trim
)
override def toString = s"$organization:$name"
}
private def attributesStr = attributes.toSeq
.sortBy { case (k, _) => k }
.map { case (k, v) => s"$k=$v" }
.mkString(";")
sealed abstract class Scope(val name: String)
override def toString =
s"$organization:$name" +
(if (attributes.nonEmpty) s";$attributesStr" else "")
}
/**
* Dependencies with the same @module will typically see their @version-s merged.
@ -35,53 +39,88 @@ sealed abstract class Scope(val name: String)
case class Dependency(
module: Module,
version: String,
scope: Scope,
attributes: Attributes,
configuration: String,
exclusions: Set[(String, String)],
// Maven-specific
attributes: Attributes,
optional: Boolean
) {
def moduleVersion = (module, version)
}
// Maven-specific
case class Attributes(
`type`: String,
classifier: String
)
) {
def publication(name: String, ext: String): Publication =
Publication(name, `type`, ext, classifier)
}
case class Project(
module: Module,
version: String,
dependencies: Seq[Dependency],
// First String is configuration (scope for Maven)
dependencies: Seq[(String, Dependency)],
// For Maven, this is the standard scopes as an Ivy configuration
configurations: Map[String, Seq[String]],
// Maven-specific
parent: Option[(Module, String)],
dependencyManagement: Seq[Dependency],
properties: Map[String, String],
dependencyManagement: Seq[(String, Dependency)],
properties: Seq[(String, String)],
profiles: Seq[Profile],
versions: Option[Versions],
snapshotVersioning: Option[SnapshotVersioning]
snapshotVersioning: Option[SnapshotVersioning],
// Ivy-specific
// First String is configuration
publications: Seq[(String, Publication)],
// Extra infos, not used during resolution
info: Info
) {
def moduleVersion = (module, version)
/** All configurations that each configuration extends, including the ones it extends transitively */
lazy val allConfigurations: Map[String, Set[String]] =
Orders.allConfigurations(configurations)
}
object Scope {
case object Compile extends Scope("compile")
case object Runtime extends Scope("runtime")
case object Test extends Scope("test")
case object Provided extends Scope("provided")
case object Import extends Scope("import")
case class Other(override val name: String) extends Scope(name)
/** Extra project info, not used during resolution */
case class Info(
description: String,
homePage: String,
licenses: Seq[(String, Option[String])],
developers: Seq[Info.Developer],
publication: Option[Versions.DateTime]
)
object Info {
case class Developer(
id: String,
name: String,
url: String
)
val empty = Info("", "", Nil, Nil, None)
}
// Maven-specific
case class Activation(properties: Seq[(String, Option[String])])
// Maven-specific
case class Profile(
id: String,
activeByDefault: Option[Boolean],
activation: Activation,
dependencies: Seq[Dependency],
dependencyManagement: Seq[Dependency],
dependencies: Seq[(String, Dependency)],
dependencyManagement: Seq[(String, Dependency)],
properties: Map[String, String]
)
// Maven-specific
case class Versions(
latest: String,
release: String,
@ -100,6 +139,7 @@ object Versions {
)
}
// Maven-specific
case class SnapshotVersion(
classifier: String,
extension: String,
@ -107,6 +147,7 @@ case class SnapshotVersion(
updated: Option[Versions.DateTime]
)
// Maven-specific
case class SnapshotVersioning(
module: Module,
version: String,
@ -119,15 +160,40 @@ case class SnapshotVersioning(
snapshotVersions: Seq[SnapshotVersion]
)
// Ivy-specific
case class Publication(
name: String,
`type`: String,
ext: String,
classifier: String
) {
def attributes: Attributes = Attributes(`type`, classifier)
}
case class Artifact(
url: String,
checksumUrls: Map[String, String],
extra: Map[String, Artifact],
attributes: Attributes
attributes: Attributes,
changing: Boolean
)
object Artifact {
trait Source {
def artifacts(dependency: Dependency, project: Project): Seq[Artifact]
def artifacts(
dependency: Dependency,
project: Project,
overrideClassifiers: Option[Seq[String]]
): Seq[Artifact]
}
object Source {
val empty: Source = new Source {
def artifacts(
dependency: Dependency,
project: Project,
overrideClassifiers: Option[Seq[String]]
): Seq[Artifact] = Nil
}
}
}

View File

@ -2,39 +2,61 @@ package coursier.core
object Orders {
/** Minimal ad-hoc partial order */
trait PartialOrder[A] {
/**
* x < y: Some(neg. integer)
* x == y: Some(0)
* x > y: Some(pos. integer)
* x, y not related: None
*/
def cmp(x: A, y: A): Option[Int]
trait PartialOrdering[T] extends scala.math.PartialOrdering[T] {
def lteq(x: T, y: T): Boolean =
tryCompare(x, y)
.exists(_ <= 0)
}
/** All configurations that each configuration extends, including the ones it extends transitively */
def allConfigurations(configurations: Map[String, Seq[String]]): Map[String, Set[String]] = {
def allParents(config: String): Set[String] = {
def helper(configs: Set[String], acc: Set[String]): Set[String] =
if (configs.isEmpty)
acc
else if (configs.exists(acc))
helper(configs -- acc, acc)
else if (configs.exists(!configurations.contains(_))) {
val (remaining, notFound) = configs.partition(configurations.contains)
helper(remaining, acc ++ notFound)
} else {
val extraConfigs = configs.flatMap(configurations)
helper(extraConfigs, acc ++ configs)
}
helper(Set(config), Set.empty)
}
configurations
.keys
.toList
.map(config => config -> (allParents(config) - config))
.toMap
}
/**
* Only relations:
* Compile < Runtime < Test
*/
implicit val mavenScopePartialOrder: PartialOrder[Scope] =
new PartialOrder[Scope] {
val higher = Map[Scope, Set[Scope]](
Scope.Compile -> Set(Scope.Runtime, Scope.Test),
Scope.Runtime -> Set(Scope.Test)
)
def configurationPartialOrder(configurations: Map[String, Seq[String]]): PartialOrdering[String] =
new PartialOrdering[String] {
val allParentsMap = allConfigurations(configurations)
def cmp(x: Scope, y: Scope) =
if (x == y) Some(0)
else if (higher.get(x).exists(_(y))) Some(-1)
else if (higher.get(y).exists(_(x))) Some(1)
else None
def tryCompare(x: String, y: String) =
if (x == y)
Some(0)
else if (allParentsMap.get(x).exists(_(y)))
Some(-1)
else if (allParentsMap.get(y).exists(_(x)))
Some(1)
else
None
}
/** Non-optional < optional */
implicit val optionalPartialOrder: PartialOrder[Boolean] =
new PartialOrder[Boolean] {
def cmp(x: Boolean, y: Boolean) =
val optionalPartialOrder: PartialOrdering[Boolean] =
new PartialOrdering[Boolean] {
def tryCompare(x: Boolean, y: Boolean) =
Some(
if (x == y) 0
else if (x) 1
@ -51,8 +73,8 @@ object Orders {
*
* In particular, no exclusions <= anything <= Set(("*", "*"))
*/
implicit val exclusionsPartialOrder: PartialOrder[Set[(String, String)]] =
new PartialOrder[Set[(String, String)]] {
val exclusionsPartialOrder: PartialOrdering[Set[(String, String)]] =
new PartialOrdering[Set[(String, String)]] {
def boolCmp(a: Boolean, b: Boolean) = (a, b) match {
case (true, true) => Some(0)
case (true, false) => Some(1)
@ -60,7 +82,7 @@ object Orders {
case (false, false) => None
}
def cmp(x: Set[(String, String)], y: Set[(String, String)]) = {
def tryCompare(x: Set[(String, String)], y: Set[(String, String)]) = {
val (xAll, xExcludeByOrg1, xExcludeByName1, xRemaining0) = Exclusions.partition(x)
val (yAll, yExcludeByOrg1, yExcludeByName1, yRemaining0) = Exclusions.partition(y)
@ -98,23 +120,44 @@ object Orders {
}
}
private def fallbackConfigIfNecessary(dep: Dependency, configs: Set[String]): Dependency =
Parse.withFallbackConfig(dep.configuration) match {
case Some((main, fallback)) =>
val config0 =
if (configs(main))
main
else if (configs(fallback))
fallback
else
dep.configuration
dep.copy(configuration = config0)
case _ =>
dep
}
/**
* Assume all dependencies have same `module`, `version`, and `artifact`; see `minDependencies`
* if they don't.
*/
def minDependenciesUnsafe(dependencies: Set[Dependency]): Set[Dependency] = {
def minDependenciesUnsafe(
dependencies: Set[Dependency],
configs: Map[String, Seq[String]]
): Set[Dependency] = {
val availableConfigs = configs.keySet
val groupedDependencies = dependencies
.groupBy(dep => (dep.optional, dep.scope))
.map(fallbackConfigIfNecessary(_, availableConfigs))
.groupBy(dep => (dep.optional, dep.configuration))
.mapValues(deps => deps.head.copy(exclusions = deps.foldLeft(Exclusions.one)((acc, dep) => Exclusions.meet(acc, dep.exclusions))))
.toList
val remove =
for {
List(((xOpt, xScope), xDep), ((yOpt, yScope), yDep)) <- groupedDependencies.combinations(2)
optCmp <- optionalPartialOrder.cmp(xOpt, yOpt).iterator
scopeCmp <- mavenScopePartialOrder.cmp(xScope, yScope).iterator
optCmp <- optionalPartialOrder.tryCompare(xOpt, yOpt).iterator
scopeCmp <- configurationPartialOrder(configs).tryCompare(xScope, yScope).iterator
if optCmp*scopeCmp >= 0
exclCmp <- exclusionsPartialOrder.cmp(xDep.exclusions, yDep.exclusions).iterator
exclCmp <- exclusionsPartialOrder.tryCompare(xDep.exclusions, yDep.exclusions).iterator
if optCmp*exclCmp >= 0
if scopeCmp*exclCmp >= 0
xIsMin = optCmp < 0 || scopeCmp < 0 || exclCmp < 0
@ -130,10 +173,13 @@ object Orders {
*
* The returned set brings exactly the same things as `dependencies`, with no redundancy.
*/
def minDependencies(dependencies: Set[Dependency]): Set[Dependency] = {
def minDependencies(
dependencies: Set[Dependency],
configs: ((Module, String)) => Map[String, Seq[String]]
): Set[Dependency] = {
dependencies
.groupBy(_.copy(scope = Scope.Other(""), exclusions = Set.empty, optional = false))
.mapValues(minDependenciesUnsafe)
.groupBy(_.copy(configuration = "", exclusions = Set.empty, optional = false))
.mapValues(deps => minDependenciesUnsafe(deps, configs(deps.head.moduleVersion)))
.valuesIterator
.fold(Set.empty)(_ ++ _)
}

View File

@ -1,18 +1,10 @@
package coursier.core
import java.util.regex.Pattern.quote
import coursier.core.compatibility._
object Parse {
def scope(s: String): Scope = s match {
case "compile" => Scope.Compile
case "runtime" => Scope.Runtime
case "test" => Scope.Test
case "provided" => Scope.Provided
case "import" => Scope.Import
case other => Scope.Other(other)
}
def version(s: String): Option[Version] = {
if (s.isEmpty || s.exists(c => c != '.' && c != '-' && c != '_' && !c.letterOrDigit)) None
else Some(Version(s))
@ -40,4 +32,20 @@ object Parse {
.orElse(versionInterval(s).map(VersionConstraint.Interval))
}
val fallbackConfigRegex = {
val noPar = "([^" + quote("()") + "]*)"
"^" + noPar + quote("(") + noPar + quote(")") + "$"
}.r
def withFallbackConfig(config: String): Option[(String, String)] =
Parse.fallbackConfigRegex.findAllMatchIn(config).toSeq match {
case Seq(m) =>
assert(m.groupCount == 2)
val main = config.substring(m.start(1), m.end(1))
val fallback = config.substring(m.start(2), m.end(2))
Some((main, fallback))
case _ =>
None
}
}

View File

@ -1,5 +1,7 @@
package coursier.core
import coursier.Fetch
import scala.language.higherKinds
import scalaz._
@ -10,7 +12,7 @@ trait Repository {
def find[F[_]](
module: Module,
version: String,
fetch: Repository.Fetch[F]
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, (Artifact.Source, Project)]
@ -18,52 +20,6 @@ trait Repository {
object Repository {
type Fetch[F[_]] = Artifact => EitherT[F, String, String]
/**
* Try to find `module` among `repositories`.
*
* Look at `repositories` from the left, one-by-one, and stop at first success.
* Else, return all errors, in the same order.
*
* The `version` field of the returned `Project` in case of success may not be
* equal to the provided one, in case the latter is not a specific
* version (e.g. version interval). Which version get chosen depends on
* the repository implementation.
*/
def find[F[_]](
repositories: Seq[Repository],
module: Module,
version: String,
fetch: Repository.Fetch[F]
)(implicit
F: Monad[F]
): EitherT[F, Seq[String], (Artifact.Source, Project)] = {
val lookups = repositories
.map(repo => repo -> repo.find(module, version, fetch).run)
val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) {
case (acc, (repo, eitherProjTask)) =>
F.bind(acc) {
case -\/(errors) =>
F.map(eitherProjTask)(_.flatMap{case (source, project) =>
if (project.module == module) \/-((source, project))
else -\/(s"Wrong module returned (expected: $module, got: ${project.module})")
}.leftMap(error => error +: errors))
case res @ \/-(_) =>
F.point(res)
}
}
EitherT(F.map(task)(_.leftMap(_.reverse)))
.map {case x @ (_, proj) =>
assert(proj.module == module)
x
}
}
implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal {
def withDefaultChecksums: Artifact =
underlying.copy(checksumUrls = underlying.checksumUrls ++ Seq(
@ -73,20 +29,15 @@ object Repository {
def withDefaultSignature: Artifact =
underlying.copy(extra = underlying.extra ++ Seq(
"sig" ->
Artifact(underlying.url + ".asc", Map.empty, Map.empty, Attributes("asc", ""))
Artifact(
underlying.url + ".asc",
Map.empty,
Map.empty,
Attributes("asc", ""),
changing = underlying.changing
)
.withDefaultChecksums
))
def withJavadocSources: Artifact = {
val base = underlying.url.stripSuffix(".jar")
underlying.copy(extra = underlying.extra ++ Seq(
"sources" -> Artifact(base + "-sources.jar", Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
.withDefaultChecksums
.withDefaultSignature,
"javadoc" -> Artifact(base + "-javadoc.jar", Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
.withDefaultChecksums
.withDefaultSignature
))
}
}
}

View File

@ -37,39 +37,33 @@ object Resolution {
(dep.module.organization, dep.module.name, dep.attributes.`type`)
def add(
dict: Map[Key, Dependency],
dep: Dependency
): Map[Key, Dependency] = {
dict: Map[Key, (String, Dependency)],
item: (String, Dependency)
): Map[Key, (String, Dependency)] = {
val key0 = key(dep)
val key0 = key(item._2)
if (dict.contains(key0))
dict
else
dict + (key0 -> dep)
dict + (key0 -> item)
}
def addSeq(
dict: Map[Key, Dependency],
deps: Seq[Dependency]
): Map[Key, Dependency] =
dict: Map[Key, (String, Dependency)],
deps: Seq[(String, Dependency)]
): Map[Key, (String, Dependency)] =
(dict /: deps)(add)
}
def mergeProperties(
dict: Map[String, String],
other: Map[String, String]
): Map[String, String] =
dict ++ other.filterKeys(!dict.contains(_))
def addDependencies(deps: Seq[Seq[Dependency]]): Seq[Dependency] = {
def addDependencies(deps: Seq[Seq[(String, Dependency)]]): Seq[(String, Dependency)] = {
val res =
(deps :\ (Set.empty[DepMgmt.Key], Seq.empty[Dependency])) {
(deps :\ (Set.empty[DepMgmt.Key], Seq.empty[(String, Dependency)])) {
case (deps0, (set, acc)) =>
val deps = deps0
.filter(dep => !set(DepMgmt.key(dep)))
.filter{case (_, dep) => !set(DepMgmt.key(dep))}
(set ++ deps.map(DepMgmt.key), acc ++ deps)
(set ++ deps.map{case (_, dep) => DepMgmt.key(dep)}, acc ++ deps)
}
res._2
@ -79,49 +73,59 @@ object Resolution {
quote("${") + "([a-zA-Z0-9-.]*)" + quote("}")
).r
def substituteProps(s: String, properties: Map[String, String]) = {
val matches = propRegex
.findAllMatchIn(s)
.toList
.reverse
if (matches.isEmpty) s
else {
val output =
(new StringBuilder(s) /: matches) { (b, m) =>
properties
.get(m.group(1))
.fold(b)(b.replace(m.start, m.end, _))
}
output.result()
}
}
def propertiesMap(props: Seq[(String, String)]): Map[String, String] =
props.foldLeft(Map.empty[String, String]) {
case (acc, (k, v0)) =>
val v = substituteProps(v0, acc)
acc + (k -> v)
}
/**
* Substitutes `properties` in `dependencies`.
*/
def withProperties(
dependencies: Seq[Dependency],
dependencies: Seq[(String, Dependency)],
properties: Map[String, String]
): Seq[Dependency] = {
): Seq[(String, Dependency)] = {
def substituteProps(s: String) = {
val matches = propRegex
.findAllMatchIn(s)
.toList
.reverse
if (matches.isEmpty) s
else {
val output =
(new StringBuilder(s) /: matches) { (b, m) =>
properties
.get(m.group(1))
.fold(b)(b.replace(m.start, m.end, _))
}
output.result()
}
}
def substituteProps0(s: String) =
substituteProps(s, properties)
dependencies
.map{ dep =>
dep.copy(
.map {case (config, dep) =>
substituteProps0(config) -> dep.copy(
module = dep.module.copy(
organization = substituteProps(dep.module.organization),
name = substituteProps(dep.module.name)
organization = substituteProps0(dep.module.organization),
name = substituteProps0(dep.module.name)
),
version = substituteProps(dep.version),
version = substituteProps0(dep.version),
attributes = dep.attributes.copy(
`type` = substituteProps(dep.attributes.`type`),
classifier = substituteProps(dep.attributes.classifier)
`type` = substituteProps0(dep.attributes.`type`),
classifier = substituteProps0(dep.attributes.classifier)
),
scope = Parse.scope(substituteProps(dep.scope.name)),
configuration = substituteProps0(dep.configuration),
exclusions = dep.exclusions
.map{case (org, name) =>
(substituteProps(org), substituteProps(name))
(substituteProps0(org), substituteProps0(name))
}
// FIXME The content of the optional tag may also be a property in
// the original POM. Maybe not parse it that earlier?
@ -165,33 +169,35 @@ object Resolution {
def merge(
dependencies: TraversableOnce[Dependency],
forceVersions: Map[Module, String]
): (Seq[Dependency], Seq[Dependency]) = {
): (Seq[Dependency], Seq[Dependency], Map[Module, String]) = {
val mergedByModVer = dependencies
.toList
.groupBy(dep => dep.module)
.map { case (module, deps) =>
module -> {
forceVersions.get(module) match {
val (versionOpt, updatedDeps) = forceVersions.get(module) match {
case None =>
if (deps.lengthCompare(1) == 0) \/-(deps)
if (deps.lengthCompare(1) == 0) (Some(deps.head.version), \/-(deps))
else {
val versions = deps
.map(_.version)
.distinct
val versionOpt = mergeVersions(versions)
versionOpt match {
(versionOpt, versionOpt match {
case Some(version) =>
\/-(deps.map(dep => dep.copy(version = version)))
case None =>
-\/(deps)
}
})
}
case Some(forcedVersion) =>
\/-(deps.map(dep => dep.copy(version = forcedVersion)))
(Some(forcedVersion), \/-(deps.map(dep => dep.copy(version = forcedVersion))))
}
(updatedDeps, versionOpt)
}
}
@ -201,33 +207,16 @@ object Resolution {
(
merged
.collect{case -\/(dep) => dep}
.collect { case (-\/(dep), _) => dep }
.flatten,
merged
.collect{case \/-(dep) => dep}
.flatten
.collect { case (\/-(dep), _) => dep }
.flatten,
mergedByModVer
.collect { case (mod, (_, Some(ver))) => mod -> ver }
)
}
/**
* If one of our dependency has scope `base`, and a transitive
* dependency of it has scope `transitive`, return the scope of
* the latter for us, if any. If empty, means the transitive dependency
* should not be considered a dependency for us.
*
* See https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Scope.
*/
def resolveScope(
base: Scope,
transitive: Scope
): Option[Scope] =
(base, transitive) match {
case (other, Scope.Compile) => Some(other)
case (Scope.Compile, Scope.Runtime) => Some(Scope.Runtime)
case (other, Scope.Runtime) => Some(other)
case _ => None
}
/**
* Applies `dependencyManagement` to `dependencies`.
*
@ -235,36 +224,39 @@ object Resolution {
* `dependencyManagement`.
*/
def depsWithDependencyManagement(
dependencies: Seq[Dependency],
dependencyManagement: Seq[Dependency]
): Seq[Dependency] = {
dependencies: Seq[(String, Dependency)],
dependencyManagement: Seq[(String, Dependency)]
): Seq[(String, Dependency)] = {
// See http://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Management
lazy val dict = DepMgmt.addSeq(Map.empty, dependencyManagement)
dependencies
.map { dep0 =>
.map {case (config0, dep0) =>
var config = config0
var dep = dep0
for (mgmtDep <- dict.get(DepMgmt.key(dep0))) {
for ((mgmtConfig, mgmtDep) <- dict.get(DepMgmt.key(dep0))) {
if (dep.version.isEmpty)
dep = dep.copy(version = mgmtDep.version)
if (dep.scope.name.isEmpty)
dep = dep.copy(scope = mgmtDep.scope)
if (config.isEmpty)
config = mgmtConfig
if (dep.exclusions.isEmpty)
dep = dep.copy(exclusions = mgmtDep.exclusions)
}
dep
(config, dep)
}
}
def withDefaultScope(dep: Dependency): Dependency =
if (dep.scope.name.isEmpty)
dep.copy(scope = Scope.Compile)
val defaultConfiguration = "compile"
def withDefaultConfig(dep: Dependency): Dependency =
if (dep.configuration.isEmpty)
dep.copy(configuration = defaultConfiguration)
else
dep
@ -272,19 +264,48 @@ object Resolution {
* Filters `dependencies` with `exclusions`.
*/
def withExclusions(
dependencies: Seq[Dependency],
dependencies: Seq[(String, Dependency)],
exclusions: Set[(String, String)]
): Seq[Dependency] = {
): Seq[(String, Dependency)] = {
val filter = Exclusions(exclusions)
dependencies
.filter(dep => filter(dep.module.organization, dep.module.name))
.map(dep =>
dep.copy(
.filter{case (_, dep) => filter(dep.module.organization, dep.module.name) }
.map{case (config, dep) =>
config -> dep.copy(
exclusions = Exclusions.minimize(dep.exclusions ++ exclusions)
)
)
}
}
def withParentConfigurations(config: String, configurations: Map[String, Seq[String]]): Set[String] = {
@tailrec
def helper(configs: Set[String], acc: Set[String]): Set[String] =
if (configs.isEmpty)
acc
else if (configs.exists(acc))
helper(configs -- acc, acc)
else if (configs.exists(!configurations.contains(_))) {
val (remaining, notFound) = configs.partition(configurations.contains)
helper(remaining, acc ++ notFound)
} else {
val extraConfigs = configs.flatMap(configurations)
helper(extraConfigs, acc ++ configs)
}
val config0 = Parse.withFallbackConfig(config) match {
case Some((main, fallback)) =>
if (configurations.contains(main))
main
else if (configurations.contains(fallback))
fallback
else
main
case None => config
}
helper(Set(config0), Set.empty)
}
/**
@ -303,38 +324,48 @@ object Resolution {
// come from parents or dependency management. This may not be
// the right thing to do.
val properties = mergeProperties(
project.properties,
Map(
"project.groupId" -> project.module.organization,
"project.artifactId" -> project.module.name,
"project.version" -> project.version
)
// FIXME The extra properties should only be added for Maven projects, not Ivy ones
val properties0 = Seq(
// some artifacts seem to require these (e.g. org.jmock:jmock-legacy:2.5.1)
// although I can find no mention of them in any manual / spec
"pom.groupId" -> project.module.organization,
"pom.artifactId" -> project.module.name,
"pom.version" -> project.version
) ++ project.properties ++ Seq(
"project.groupId" -> project.module.organization,
"project.artifactId" -> project.module.name,
"project.version" -> project.version
)
val deps =
withExclusions(
depsWithDependencyManagement(
// Important: properties have to be applied to both,
// so that dep mgmt can be matched properly
// Tested with org.ow2.asm:asm-commons:5.0.2 in CentralTests
withProperties(project.dependencies, properties),
withProperties(project.dependencyManagement, properties)
),
from.exclusions
)
.map(withDefaultScope)
val properties = propertiesMap(properties0)
deps
.flatMap { trDep =>
resolveScope(from.scope, trDep.scope)
.map(scope =>
trDep.copy(
scope = scope,
optional = trDep.optional || from.optional
)
)
}
val configurations = withParentConfigurations(from.configuration, project.configurations)
withExclusions(
depsWithDependencyManagement(
// Important: properties have to be applied to both,
// so that dep mgmt can be matched properly
// Tested with org.ow2.asm:asm-commons:5.0.2 in CentralTests
withProperties(project.dependencies, properties),
withProperties(project.dependencyManagement, properties)
),
from.exclusions
)
.map{
case (config, dep) =>
(if (config.isEmpty) defaultConfiguration else config) -> {
if (dep.configuration.isEmpty)
dep.copy(configuration = defaultConfiguration)
else
dep
}
}
.collect{case (config, dep) if configurations(config) =>
if (from.optional)
dep.copy(optional = true)
else
dep
}
}
/**
@ -432,13 +463,13 @@ case class Resolution(
* May contain dependencies added in previous iterations, but no more
* required. These are filtered below, see `newDependencies`.
*
* Returns a tuple made of the conflicting dependencies, and all
* the dependencies.
* Returns a tuple made of the conflicting dependencies, all
* the dependencies, and the retained version of each module.
*/
def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) =
def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency], Map[Module, String]) =
// TODO Provide the modules whose version was forced by dependency overrides too
merge(
rootDependencies.map(withDefaultScope) ++ dependencies ++ transitiveDependencies,
rootDependencies.map(withDefaultConfig) ++ dependencies ++ transitiveDependencies,
forceVersions
)
@ -461,7 +492,7 @@ case class Resolution(
*/
def isDone: Boolean = {
def isFixPoint = {
val (nextConflicts, _) = nextDependenciesAndConflicts
val (nextConflicts, _, _) = nextDependenciesAndConflicts
dependencies == (newDependencies ++ nextConflicts) &&
conflicts == nextConflicts.toSet
@ -480,7 +511,7 @@ case class Resolution(
* The versions of all the dependencies returned are erased (emptied).
*/
def reverseDependencies: Map[Dependency, Vector[Dependency]] = {
val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts
val (updatedConflicts, updatedDeps, _) = nextDependenciesAndConflicts
val trDepsSeq =
for {
@ -507,7 +538,7 @@ case class Resolution(
*/
def remainingDependencies: Set[Dependency] = {
val rootDependencies0 = rootDependencies
.map(withDefaultScope)
.map(withDefaultConfig)
.map(eraseVersion)
@tailrec
@ -549,7 +580,7 @@ case class Resolution(
}
private def nextNoMissingUnsafe: Resolution = {
val (newConflicts, _) = nextDependenciesAndConflicts
val (newConflicts, _, _) = nextDependenciesAndConflicts
copy(
dependencies = newDependencies ++ newConflicts,
@ -583,24 +614,31 @@ case class Resolution(
project: Project
): Set[ModuleVersion] = {
val approxProperties =
val approxProperties0 =
project.parent
.flatMap(projectCache.get)
.map(_._2.properties)
.fold(project.properties)(mergeProperties(project.properties, _))
.fold(project.properties)(project.properties ++ _)
val approxProperties = propertiesMap(approxProperties0) ++ Seq(
"project.groupId" -> project.module.organization,
"project.artifactId" -> project.module.name,
"project.version" -> project.version
)
val profileDependencies =
profiles(
project,
approxProperties,
profileActivation getOrElse defaultProfileActivation
).flatMap(_.dependencies)
).flatMap(p => p.dependencies ++ p.dependencyManagement)
val modules =
(project.dependencies ++ profileDependencies)
.collect{
case dep if dep.scope == Scope.Import => dep.moduleVersion
}
val modules = withProperties(
project.dependencies ++ project.dependencyManagement ++ profileDependencies,
approxProperties
).collect {
case ("import", dep) => dep.moduleVersion
}
modules.toSet ++ project.parent
}
@ -657,11 +695,21 @@ case class Resolution(
*/
def withDependencyManagement(project: Project): Project = {
val approxProperties =
// A bit fragile, but seems to work
// TODO Add non regression test for the touchy org.glassfish.jersey.core:jersey-client:2.19
// (for the way it uses org.glassfish.hk2:hk2-bom,2.4.0-b25)
val approxProperties0 =
project.parent
.filter(projectCache.contains)
.map(projectCache(_)._2.properties)
.fold(project.properties)(mergeProperties(project.properties, _))
.map(projectCache(_)._2.properties.toMap)
.fold(project.properties)(project.properties ++ _)
val approxProperties = propertiesMap(approxProperties0) ++ Seq(
"project.groupId" -> project.module.organization,
"project.artifactId" -> project.module.name,
"project.version" -> project.version
)
val profiles0 = profiles(
project,
@ -670,20 +718,29 @@ case class Resolution(
)
val dependencies0 = addDependencies(
project.dependencies +: profiles0.map(_.dependencies)
(project.dependencies +: profiles0.map(_.dependencies)).map(withProperties(_, approxProperties))
)
val dependenciesMgmt0 = addDependencies(
(project.dependencyManagement +: profiles0.map(_.dependencyManagement)).map(withProperties(_, approxProperties))
)
val properties0 =
(project.properties /: profiles0) { (acc, p) =>
mergeProperties(acc, p.properties)
acc ++ p.properties
}
val deps = (
val deps0 = (
dependencies0
.collect { case dep if dep.scope == Scope.Import =>
.collect { case ("import", dep) =>
dep.moduleVersion
} ++
dependenciesMgmt0
.collect { case ("import", dep) =>
dep.moduleVersion
} ++
project.parent
).filter(projectCache.contains)
)
val deps = deps0.filter(projectCache.contains)
val projs = deps
.map(projectCache(_)._2)
@ -693,41 +750,75 @@ case class Resolution(
profiles0.map(_.dependencyManagement) ++
projs.map(_.dependencyManagement)
)
).foldLeft(Map.empty[DepMgmt.Key, Dependency])(DepMgmt.addSeq)
)
.map(withProperties(_, approxProperties))
.foldLeft(Map.empty[DepMgmt.Key, (String, Dependency)])(DepMgmt.addSeq)
val depsSet = deps.toSet
project.copy(
dependencies =
dependencies0
.filterNot(dep =>
dep.scope == Scope.Import && depsSet(dep.moduleVersion)
) ++
.filterNot{case (config, dep) =>
config == "import" && depsSet(dep.moduleVersion)
} ++
project.parent
.filter(projectCache.contains)
.toSeq
.flatMap(projectCache(_)._2.dependencies),
dependencyManagement = depMgmt.values.toSeq,
dependencyManagement = depMgmt.values.toSeq
.filterNot{case (config, dep) =>
config == "import" && depsSet(dep.moduleVersion)
},
properties = project.parent
.filter(projectCache.contains)
.map(projectCache(_)._2.properties)
.fold(properties0)(mergeProperties(properties0, _))
.fold(properties0)(properties0 ++ _)
)
}
def minDependencies: Set[Dependency] =
Orders.minDependencies(dependencies)
Orders.minDependencies(
dependencies,
dep =>
projectCache
.get(dep)
.map(_._2.configurations)
.getOrElse(Map.empty)
)
def artifacts: Seq[Artifact] =
private def artifacts0(overrideClassifiers: Option[Seq[String]]): Seq[Artifact] =
for {
dep <- minDependencies.toSeq
(source, proj) <- projectCache
.get(dep.moduleVersion)
.toSeq
artifact <- source
.artifacts(dep, proj)
.artifacts(dep, proj, overrideClassifiers)
} yield artifact
def classifiersArtifacts(classifiers: Seq[String]): Seq[Artifact] =
artifacts0(Some(classifiers))
def artifacts: Seq[Artifact] =
artifacts0(None)
private def dependencyArtifacts0(overrideClassifiers: Option[Seq[String]]): Seq[(Dependency, Artifact)] =
for {
dep <- minDependencies.toSeq
(source, proj) <- projectCache
.get(dep.moduleVersion)
.toSeq
artifact <- source
.artifacts(dep, proj, overrideClassifiers)
} yield dep -> artifact
def dependencyArtifacts: Seq[(Dependency, Artifact)] =
dependencyArtifacts0(None)
def dependencyClassifiersArtifacts(classifiers: Seq[String]): Seq[(Dependency, Artifact)] =
dependencyArtifacts0(Some(classifiers))
def errors: Seq[(Dependency, Seq[String])] =
for {
dep <- dependencies.toSeq
@ -735,4 +826,30 @@ case class Resolution(
.get(dep.moduleVersion)
.toSeq
} yield (dep, err)
def subset(dependencies: Set[Dependency]): Resolution = {
val (_, _, finalVersions) = nextDependenciesAndConflicts
def updateVersion(dep: Dependency): Dependency =
dep.copy(version = finalVersions.getOrElse(dep.module, dep.version))
@tailrec def helper(current: Set[Dependency]): Set[Dependency] = {
val newDeps = current ++ current
.flatMap(finalDependencies0)
.map(updateVersion)
val anyNewDep = (newDeps -- current).nonEmpty
if (anyNewDep)
helper(newDeps)
else
newDeps
}
copy(
rootDependencies = dependencies,
dependencies = helper(dependencies.map(updateVersion))
// don't know if something should be done about conflicts
)
}
}

View File

@ -7,7 +7,7 @@ import scala.annotation.tailrec
sealed trait ResolutionProcess {
def run[F[_]](
fetch: ResolutionProcess.Fetch[F],
fetch: Fetch.Metadata[F],
maxIterations: Int = -1
)(implicit
F: Monad[F]
@ -34,7 +34,7 @@ sealed trait ResolutionProcess {
}
def next[F[_]](
fetch: ResolutionProcess.Fetch[F]
fetch: Fetch.Metadata[F]
)(implicit
F: Monad[F]
): F[ResolutionProcess] = {
@ -58,7 +58,7 @@ case class Missing(
cont: Resolution => ResolutionProcess
) extends ResolutionProcess {
def next(results: ResolutionProcess.FetchResult): ResolutionProcess = {
def next(results: Fetch.MD): ResolutionProcess = {
val errors = results
.collect{case (modVer, -\/(errs)) => modVer -> errs }
@ -72,7 +72,7 @@ case class Missing(
val depMgmtMissing = depMgmtMissing0 -- results.map(_._1)
def cont0(res: Resolution) = {
val res0 =
val res0 =
successes.foldLeft(res){case (acc, (modVer, (source, proj))) =>
acc.copy(projectCache = acc.projectCache + (
modVer -> (source, acc.withDependencyManagement(proj))
@ -121,12 +121,5 @@ object ResolutionProcess {
else
Missing(resolution0.missingFromCache.toSeq, resolution0, apply)
}
type FetchResult = Seq[(
(Module, String),
Seq[String] \/ (Artifact.Source, Project)
)]
type Fetch[F[_]] = Seq[(Module, String)] => F[FetchResult]
}

View File

@ -0,0 +1,272 @@
package coursier.ivy
import coursier.Fetch
import coursier.core._
import scala.annotation.tailrec
import scala.util.matching.Regex
import scalaz._
import java.util.regex.Pattern.quote
object IvyRepository {
val optionalPartRegex = (quote("(") + "[^" + quote("{()}") + "]*" + quote(")")).r
val variableRegex = (quote("[") + "[^" + quote("{[()]}") + "]*" + quote("]")).r
val propertyRegex = (quote("${") + "[^" + quote("{[()]}") + "]*" + quote("}")).r
sealed abstract class PatternPart(val effectiveStart: Int, val effectiveEnd: Int) extends Product with Serializable {
require(effectiveStart <= effectiveEnd)
def start = effectiveStart
def end = effectiveEnd
// FIXME Some kind of validation should be used here, to report all the missing variables,
// not only the first one missing.
def apply(content: String): Map[String, String] => String \/ String
}
object PatternPart {
case class Literal(override val effectiveStart: Int, override val effectiveEnd: Int) extends PatternPart(effectiveStart, effectiveEnd) {
def apply(content: String): Map[String, String] => String \/ String = {
assert(content.length == effectiveEnd - effectiveStart)
val matches = variableRegex.findAllMatchIn(content).toList
variables =>
@tailrec
def helper(idx: Int, matches: List[Regex.Match], b: StringBuilder): String \/ String =
if (idx >= content.length)
\/-(b.result())
else {
assert(matches.headOption.forall(_.start >= idx))
matches.headOption.filter(_.start == idx) match {
case Some(m) =>
val variableName = content.substring(m.start + 1, m.end - 1)
variables.get(variableName) match {
case None => -\/(s"Variable not found: $variableName")
case Some(value) =>
b ++= value
helper(m.end, matches.tail, b)
}
case None =>
val nextIdx = matches.headOption.fold(content.length)(_.start)
b ++= content.substring(idx, nextIdx)
helper(nextIdx, matches, b)
}
}
helper(0, matches, new StringBuilder)
}
}
case class Optional(start0: Int, end0: Int) extends PatternPart(start0 + 1, end0 - 1) {
override def start = start0
override def end = end0
def apply(content: String): Map[String, String] => String \/ String = {
assert(content.length == effectiveEnd - effectiveStart)
val inner = Literal(effectiveStart, effectiveEnd).apply(content)
variables =>
\/-(inner(variables).fold(_ => "", x => x))
}
}
}
def substituteProperties(s: String, properties: Map[String, String]): String =
propertyRegex.findAllMatchIn(s).toVector.foldRight(s) { case (m, s0) =>
val key = s0.substring(m.start + "${".length, m.end - "}".length)
val value = properties.getOrElse(key, "")
s0.take(m.start) + value + s0.drop(m.end)
}
}
case class IvyRepository(
pattern: String,
changing: Option[Boolean] = None,
properties: Map[String, String] = Map.empty,
withChecksums: Boolean = true,
withSignatures: Boolean = true,
withArtifacts: Boolean = true
) extends Repository {
import Repository._
import IvyRepository._
private val pattern0 = substituteProperties(pattern, properties)
val parts = {
val optionalParts = optionalPartRegex.findAllMatchIn(pattern0).toList.map { m =>
PatternPart.Optional(m.start, m.end)
}
val len = pattern0.length
@tailrec
def helper(
idx: Int,
opt: List[PatternPart.Optional],
acc: List[PatternPart]
): Vector[PatternPart] =
if (idx >= len)
acc.toVector.reverse
else
opt match {
case Nil =>
helper(len, Nil, PatternPart.Literal(idx, len) :: acc)
case (opt0 @ PatternPart.Optional(start0, end0)) :: rem =>
if (idx < start0)
helper(start0, opt, PatternPart.Literal(idx, start0) :: acc)
else {
assert(idx == start0, s"idx: $idx, start0: $start0")
helper(end0, rem, opt0 :: acc)
}
}
helper(0, optionalParts, Nil)
}
assert(pattern0.isEmpty == parts.isEmpty)
if (pattern0.nonEmpty) {
for ((a, b) <- parts.zip(parts.tail))
assert(a.end == b.start)
assert(parts.head.start == 0)
assert(parts.last.end == pattern0.length)
}
private val substituteHelpers = parts.map { part =>
part(pattern0.substring(part.effectiveStart, part.effectiveEnd))
}
def substitute(variables: Map[String, String]): String \/ String =
substituteHelpers.foldLeft[String \/ String](\/-("")) {
case (acc0, helper) =>
for {
acc <- acc0
s <- helper(variables)
} yield acc + s
}
// See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a
// list of variables that should be supported.
// Some are missing (branch, conf, originalName).
private def variables(
module: Module,
version: String,
`type`: String,
artifact: String,
ext: String,
classifierOpt: Option[String]
) =
Map(
"organization" -> module.organization,
"organisation" -> module.organization,
"orgPath" -> module.organization.replace('.', '/'),
"module" -> module.name,
"revision" -> version,
"type" -> `type`,
"artifact" -> artifact,
"ext" -> ext
) ++ module.attributes ++ classifierOpt.map("classifier" -> _).toSeq
val source: Artifact.Source =
if (withArtifacts)
new Artifact.Source {
def artifacts(
dependency: Dependency,
project: Project,
overrideClassifiers: Option[Seq[String]]
) = {
val retained =
overrideClassifiers match {
case None =>
project.publications.collect {
case (conf, p)
if conf == "*" ||
conf == dependency.configuration ||
project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) =>
p
}
case Some(classifiers) =>
val classifiersSet = classifiers.toSet
project.publications.collect {
case (_, p) if classifiersSet(p.classifier) =>
p
}
}
val retainedWithUrl = retained.flatMap { p =>
substitute(variables(
dependency.module,
dependency.version,
p.`type`,
p.name,
p.ext,
Some(p.classifier).filter(_.nonEmpty)
)).toList.map(p -> _)
}
retainedWithUrl.map { case (p, url) =>
var artifact = Artifact(
url,
Map.empty,
Map.empty,
p.attributes,
changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable
)
if (withChecksums)
artifact = artifact.withDefaultChecksums
if (withSignatures)
artifact = artifact.withDefaultSignature
artifact
}
}
}
else
Artifact.Source.empty
def find[F[_]](
module: Module,
version: String,
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, (Artifact.Source, Project)] = {
val eitherArtifact: String \/ Artifact =
for {
url <- substitute(
variables(module, version, "ivy", "ivy", "xml", None)
)
} yield {
var artifact = Artifact(
url,
Map.empty,
Map.empty,
Attributes("ivy", ""),
changing = changing.getOrElse(version.contains("-SNAPSHOT"))
)
if (withChecksums)
artifact = artifact.withDefaultChecksums
if (withSignatures)
artifact = artifact.withDefaultSignature
artifact
}
for {
artifact <- EitherT(F.point(eitherArtifact))
ivy <- fetch(artifact)
proj <- EitherT(F.point {
for {
xml <- \/.fromEither(compatibility.xmlParse(ivy))
_ <- if (xml.label == "ivy-module") \/-(()) else -\/("Module definition not found")
proj <- IvyXml.project(xml)
} yield proj
})
} yield (source, proj)
}
}

View File

@ -0,0 +1,172 @@
package coursier.ivy
import coursier.core._
import coursier.util.Xml._
import scalaz.{ Node => _, _ }, Scalaz._
object IvyXml {
val attributesNamespace = "http://ant.apache.org/ivy/extra"
private def info(node: Node): String \/ (Module, String) =
for {
org <- node.attribute("organisation")
name <- node.attribute("module")
version <- node.attribute("revision")
attr = node.attributesFromNamespace(attributesNamespace)
} yield (Module(org, name, attr.toMap), version)
// FIXME Errors are ignored here
private def configurations(node: Node): Seq[(String, Seq[String])] =
node.children
.filter(_.label == "conf")
.flatMap { node =>
node.attribute("name").toOption.toSeq.map(_ -> node)
}
.map { case (name, node) =>
name -> node.attribute("extends").toOption.toSeq.flatMap(_.split(','))
}
// FIXME "default(compile)" likely not to be always the default
def mappings(mapping: String): Seq[(String, String)] =
mapping.split(';').flatMap { m =>
val (froms, tos) = m.split("->", 2) match {
case Array(from) => (from, "default(compile)")
case Array(from, to) => (from, to)
}
for {
from <- froms.split(',')
to <- tos.split(',')
} yield (from.trim, to.trim)
}
// FIXME Errors ignored as above - warnings should be reported at least for anything suspicious
private def dependencies(node: Node): Seq[(String, Dependency)] =
node.children
.filter(_.label == "dependency")
.flatMap { node =>
// artifact and include sub-nodes are ignored here
val excludes = node.children
.filter(_.label == "exclude")
.flatMap { node0 =>
val org = node.attribute("org").getOrElse("*")
val name = node.attribute("module").orElse(node.attribute("name")).getOrElse("*")
val confs = node.attribute("conf").toOption.fold(Seq("*"))(_.split(','))
confs.map(_ -> (org, name))
}
.groupBy { case (conf, _) => conf }
.map { case (conf, l) => conf -> l.map { case (_, e) => e }.toSet }
val allConfsExcludes = excludes.getOrElse("*", Set.empty)
for {
org <- node.attribute("org").toOption.toSeq
name <- node.attribute("name").toOption.toSeq
version <- node.attribute("rev").toOption.toSeq
rawConf <- node.attribute("conf").toOption.toSeq
(fromConf, toConf) <- mappings(rawConf)
attr = node.attributesFromNamespace(attributesNamespace)
} yield fromConf -> Dependency(
Module(org, name, attr.toMap),
version,
toConf,
allConfsExcludes ++ excludes.getOrElse(fromConf, Set.empty),
Attributes("jar", ""), // should come from possible artifact nodes
optional = false
)
}
private def publications(node: Node): Map[String, Seq[Publication]] =
node.children
.filter(_.label == "artifact")
.flatMap { node =>
val name = node.attribute("name").getOrElse("")
val type0 = node.attribute("type").getOrElse("jar")
val ext = node.attribute("ext").getOrElse(type0)
val confs = node.attribute("conf").toOption.fold(Seq("*"))(_.split(','))
val classifier = node.attribute("classifier").toOption.getOrElse("")
confs.map(_ -> Publication(name, type0, ext, classifier))
}
.groupBy { case (conf, _) => conf }
.map { case (conf, l) => conf -> l.map { case (_, p) => p } }
def project(node: Node): String \/ Project =
for {
infoNode <- node.children
.find(_.label == "info")
.toRightDisjunction("Info not found")
(module, version) <- info(infoNode)
dependenciesNodeOpt = node.children
.find(_.label == "dependencies")
dependencies0 = dependenciesNodeOpt.map(dependencies).getOrElse(Nil)
configurationsNodeOpt = node.children
.find(_.label == "configurations")
configurationsOpt = configurationsNodeOpt.map(configurations)
configurations0 = configurationsOpt.getOrElse(Seq("default" -> Seq.empty[String]))
publicationsNodeOpt = node.children
.find(_.label == "publications")
publicationsOpt = publicationsNodeOpt.map(publications)
} yield {
val description = infoNode.children
.find(_.label == "description")
.map(_.textContent.trim)
.getOrElse("")
val licenses = infoNode.children
.filter(_.label == "license")
.flatMap { n =>
n.attribute("name").toOption.map { name =>
(name, n.attribute("url").toOption)
}.toSeq
}
val publicationDate = infoNode.attribute("publication")
.toOption
.flatMap(parseDateTime)
Project(
module,
version,
dependencies0,
configurations0.toMap,
None,
Nil,
Nil,
Nil,
None,
None,
if (publicationsOpt.isEmpty)
// no publications node -> default JAR artifact
Seq("*" -> Publication(module.name, "jar", "jar", ""))
else {
// publications node is there -> only its content (if it is empty, no artifacts,
// as per the Ivy manual)
val inAllConfs = publicationsOpt.flatMap(_.get("*")).getOrElse(Nil)
configurations0.flatMap { case (conf, _) =>
(publicationsOpt.flatMap(_.get(conf)).getOrElse(Nil) ++ inAllConfs).map(conf -> _)
}
},
Info(
description,
"",
licenses,
Nil,
publicationDate
)
)
}
}

View File

@ -1,5 +1,6 @@
package coursier.maven
import coursier.Fetch
import coursier.core._
import coursier.core.compatibility.encodeURIComponent
@ -10,6 +11,7 @@ object MavenRepository {
def ivyLikePath(
org: String,
dirName: String,
name: String,
version: String,
subDir: String,
@ -18,7 +20,7 @@ object MavenRepository {
) =
Seq(
org,
name,
dirName,
version,
subDir,
s"$name$baseSuffix.$ext"
@ -35,18 +37,40 @@ object MavenRepository {
.map(_.value)
.filter(_.nonEmpty)
val defaultConfigurations = Map(
"compile" -> Seq.empty,
"runtime" -> Seq("compile"),
"default" -> Seq("runtime"),
"test" -> Seq("runtime")
)
def dirModuleName(module: Module, sbtAttrStub: Boolean): String =
if (sbtAttrStub) {
var name = module.name
for (scalaVersion <- module.attributes.get("scalaVersion"))
name = name + "_" + scalaVersion
for (sbtVersion <- module.attributes.get("sbtVersion"))
name = name + "_" + sbtVersion
name
} else
module.name
}
case class MavenRepository(
root: String,
ivyLike: Boolean = false
ivyLike: Boolean = false,
changing: Option[Boolean] = None,
/** Hackish hack for sbt plugins mainly - what this does really sucks */
sbtAttrStub: Boolean = false
) extends Repository {
import Repository._
import MavenRepository._
val root0 = if (root.endsWith("/")) root else root + "/"
val source = MavenSource(root0, ivyLike)
val source = MavenSource(root0, ivyLike, changing, sbtAttrStub)
def projectArtifact(
module: Module,
@ -58,6 +82,7 @@ case class MavenRepository(
if (ivyLike)
ivyLikePath(
module.organization,
dirModuleName(module, sbtAttrStub), // maybe not what we should do here, don't know
module.name,
versioningValue getOrElse version,
"poms",
@ -66,7 +91,7 @@ case class MavenRepository(
)
else
module.organization.split('.').toSeq ++ Seq(
module.name,
dirModuleName(module, sbtAttrStub),
version,
s"${module.name}-${versioningValue getOrElse version}.pom"
)
@ -76,7 +101,8 @@ case class MavenRepository(
root0 + path.mkString("/"),
Map.empty,
Map.empty,
Attributes("pom", "")
Attributes("pom", ""),
changing = changing.getOrElse(version.contains("-SNAPSHOT"))
)
.withDefaultChecksums
.withDefaultSignature
@ -87,7 +113,7 @@ case class MavenRepository(
else {
val path = (
module.organization.split('.').toSeq ++ Seq(
module.name,
dirModuleName(module, sbtAttrStub),
"maven-metadata.xml"
)
) .map(encodeURIComponent)
@ -97,10 +123,11 @@ case class MavenRepository(
root0 + path.mkString("/"),
Map.empty,
Map.empty,
Attributes("pom", "")
Attributes("pom", ""),
changing = true
)
.withDefaultChecksums
.withDefaultChecksums
.withDefaultSignature
Some(artifact)
}
@ -113,7 +140,7 @@ case class MavenRepository(
else {
val path = (
module.organization.split('.').toSeq ++ Seq(
module.name,
dirModuleName(module, sbtAttrStub),
version,
"maven-metadata.xml"
)
@ -124,7 +151,8 @@ case class MavenRepository(
root0 + path.mkString("/"),
Map.empty,
Map.empty,
Attributes("pom", "")
Attributes("pom", ""),
changing = true
)
.withDefaultChecksums
.withDefaultSignature
@ -134,7 +162,7 @@ case class MavenRepository(
def versions[F[_]](
module: Module,
fetch: Repository.Fetch[F]
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, Versions] =
@ -156,7 +184,7 @@ case class MavenRepository(
def snapshotVersioning[F[_]](
module: Module,
version: String,
fetch: Repository.Fetch[F]
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, SnapshotVersioning] = {
@ -180,7 +208,7 @@ case class MavenRepository(
def findNoInterval[F[_]](
module: Module,
version: String,
fetch: Repository.Fetch[F]
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, Project] =
@ -219,7 +247,7 @@ case class MavenRepository(
module: Module,
version: String,
versioningValue: Option[String],
fetch: Repository.Fetch[F]
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, Project] = {
@ -231,7 +259,14 @@ case class MavenRepository(
xml <- \/.fromEither(compatibility.xmlParse(str))
_ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found")
proj <- Pom.project(xml)
} yield proj): (String \/ Project)
} yield proj.copy(
configurations = defaultConfigurations,
publications = Seq(
"compile" -> Publication(module.name, "jar", "jar", ""),
"docs" -> Publication(module.name, "doc", "jar", "javadoc"),
"sources" -> Publication(module.name, "src", "jar", "sources")
)
)): (String \/ Project)
}
}
}
@ -240,7 +275,7 @@ case class MavenRepository(
def find[F[_]](
module: Module,
version: String,
fetch: Repository.Fetch[F]
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, (Artifact.Source, Project)] = {

View File

@ -2,78 +2,114 @@ package coursier.maven
import coursier.core._
case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
case class MavenSource(
root: String,
ivyLike: Boolean,
changing: Option[Boolean] = None,
/** See doc on MavenRepository */
sbtAttrStub: Boolean
) extends Artifact.Source {
import Repository._
import MavenRepository._
private implicit class DocSourcesArtifactExtensions(val underlying: Artifact) {
def withJavadocSources: Artifact = {
val base = underlying.url.stripSuffix(".jar")
underlying.copy(extra = underlying.extra ++ Seq(
"sources" -> Artifact(
base + "-sources.jar",
Map.empty,
Map.empty,
Attributes("jar", "src"), // Are these the right attributes?
changing = underlying.changing
)
.withDefaultChecksums
.withDefaultSignature,
"javadoc" -> Artifact(
base + "-javadoc.jar",
Map.empty,
Map.empty,
Attributes("jar", "javadoc"), // Same comment as above
changing = underlying.changing
)
.withDefaultChecksums
.withDefaultSignature
))
}
}
def artifacts(
dependency: Dependency,
project: Project
project: Project,
overrideClassifiers: Option[Seq[String]]
): Seq[Artifact] = {
def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) =
ivyLikePath(
dependency.module.organization,
dependency.module.name,
project.version,
subDir,
baseSuffix,
ext
)
val path =
if (ivyLike)
ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`)
else {
val versioning =
project
.snapshotVersioning
.flatMap(versioning =>
mavenVersioning(versioning, dependency.attributes.classifier, dependency.attributes.`type`)
)
dependency.module.organization.split('.').toSeq ++ Seq(
dependency.module.name,
def artifactOf(module: Module, publication: Publication) = {
def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) =
ivyLikePath(
module.organization,
MavenRepository.dirModuleName(module, sbtAttrStub),
module.name,
project.version,
s"${dependency.module.name}-${versioning getOrElse project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}"
subDir,
baseSuffix,
ext
)
val path =
if (ivyLike)
ivyLikePath0(publication.`type` + "s", "", publication.ext)
else {
val versioning =
project
.snapshotVersioning
.flatMap(versioning =>
mavenVersioning(versioning, publication.classifier, publication.`type`)
)
module.organization.split('.').toSeq ++ Seq(
MavenRepository.dirModuleName(module, sbtAttrStub),
project.version,
s"${module.name}-${versioning getOrElse project.version}${Some(publication.classifier).filter(_.nonEmpty).map("-" + _).mkString}.${publication.ext}"
)
}
val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT"))
var artifact =
Artifact(
root + path.mkString("/"),
Map.empty,
Map.empty,
publication.attributes,
changing = changing0
)
.withDefaultChecksums
if (publication.ext == "jar") {
artifact = artifact.withDefaultSignature
}
var artifact =
Artifact(
root + path.mkString("/"),
Map.empty,
Map.empty,
dependency.attributes
)
.withDefaultChecksums
if (dependency.attributes.`type` == "jar") {
artifact = artifact.withDefaultSignature
// FIXME Snapshot versioning of sources and javadoc is not taken into account here.
// Will be ok if it's the same as the main JAR though.
artifact =
if (ivyLike) {
val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/")
val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/")
artifact
.copy(
extra = artifact.extra ++ Map(
"sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
.withDefaultChecksums
.withDefaultSignature,
"javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
.withDefaultChecksums
.withDefaultSignature
))
} else
artifact
.withJavadocSources
artifact
}
Seq(artifact)
overrideClassifiers match {
case Some(classifiers) =>
val classifiersSet = classifiers.toSet
project.publications.collect {
case (_, p) if classifiersSet(p.classifier) =>
artifactOf(dependency.module, p)
}
case None =>
Seq(
artifactOf(
dependency.module,
dependency.attributes.publication(
dependency.module.name,
dependency.attributes.`type`
)
)
)
}
}
}

View File

@ -7,21 +7,6 @@ import scalaz._
object Pom {
import coursier.util.Xml._
object Text {
def unapply(n: Node): Option[String] =
if (n.isText) Some(n.textContent)
else None
}
private def text(elem: Node, label: String, description: String) = {
import Scalaz.ToOptionOpsFromOption
elem.child
.find(_.label == label)
.flatMap(_.child.collectFirst{case Text(t) => t})
.toRightDisjunction(s"$description not found")
}
def property(elem: Node): String \/ (String, String) = {
// Not matching with Text, which fails on scala-js if the property value has xml comments
if (elem.isElement) \/-(elem.label -> elem.textContent)
@ -37,39 +22,37 @@ object Pom {
else e
}
name <- text(node, "artifactId", "Name")
} yield Module(organization, name).trim
} yield Module(organization, name, Map.empty).trim
}
private def readVersion(node: Node) =
text(node, "version", "Version").getOrElse("").trim
private val defaultScope = Scope.Other("")
private val defaultType = "jar"
private val defaultClassifier = ""
def dependency(node: Node): String \/ Dependency = {
def dependency(node: Node): String \/ (String, Dependency) = {
for {
mod <- module(node)
version0 = readVersion(node)
scopeOpt = text(node, "scope", "").toOption
.map(Parse.scope)
typeOpt = text(node, "type", "").toOption
classifierOpt = text(node, "classifier", "").toOption
xmlExclusions = node.child
xmlExclusions = node.children
.find(_.label == "exclusions")
.map(_.child.filter(_.label == "exclusion"))
.map(_.children.filter(_.label == "exclusion"))
.getOrElse(Seq.empty)
exclusions <- {
import Scalaz._
xmlExclusions.toList.traverseU(module(_))
}
optional = text(node, "optional", "").toOption.toSeq.contains("true")
} yield Dependency(
} yield scopeOpt.getOrElse("") -> Dependency(
mod,
version0,
scopeOpt getOrElse defaultScope,
Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier),
"",
exclusions.map(mod => (mod.organization, mod.name)).toSet,
Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier),
optional
)
}
@ -82,7 +65,7 @@ object Pom {
case _ => None
}
val properties = node.child
val properties = node.children
.filter(_.label == "property")
.flatMap{ p =>
for{
@ -99,28 +82,28 @@ object Pom {
val id = text(node, "id", "Profile ID").getOrElse("")
val xmlActivationOpt = node.child
val xmlActivationOpt = node.children
.find(_.label == "activation")
val (activeByDefault, activation) = xmlActivationOpt.fold((Option.empty[Boolean], Activation(Nil)))(profileActivation)
val xmlDeps = node.child
val xmlDeps = node.children
.find(_.label == "dependencies")
.map(_.child.filter(_.label == "dependency"))
.map(_.children.filter(_.label == "dependency"))
.getOrElse(Seq.empty)
for {
deps <- xmlDeps.toList.traverseU(dependency)
xmlDepMgmts = node.child
xmlDepMgmts = node.children
.find(_.label == "dependencyManagement")
.flatMap(_.child.find(_.label == "dependencies"))
.map(_.child.filter(_.label == "dependency"))
.flatMap(_.children.find(_.label == "dependencies"))
.map(_.children.filter(_.label == "dependency"))
.getOrElse(Seq.empty)
depMgmts <- xmlDepMgmts.toList.traverseU(dependency)
xmlProperties = node.child
xmlProperties = node.children
.find(_.label == "properties")
.map(_.child.collect{case elem if elem.isElement => elem})
.map(_.children.collect{case elem if elem.isElement => elem})
.getOrElse(Seq.empty)
properties <- {
@ -138,7 +121,7 @@ object Pom {
projModule <- module(pom, groupIdIsOptional = true)
projVersion = readVersion(pom)
parentOpt = pom.child
parentOpt = pom.children
.find(_.label == "parent")
parentModuleOpt <- parentOpt
.map(module(_).map(Some(_)))
@ -146,16 +129,16 @@ object Pom {
parentVersionOpt = parentOpt
.map(readVersion)
xmlDeps = pom.child
xmlDeps = pom.children
.find(_.label == "dependencies")
.map(_.child.filter(_.label == "dependency"))
.map(_.children.filter(_.label == "dependency"))
.getOrElse(Seq.empty)
deps <- xmlDeps.toList.traverseU(dependency)
xmlDepMgmts = pom.child
xmlDepMgmts = pom.children
.find(_.label == "dependencyManagement")
.flatMap(_.child.find(_.label == "dependencies"))
.map(_.child.filter(_.label == "dependency"))
.flatMap(_.children.find(_.label == "dependencies"))
.map(_.children.filter(_.label == "dependency"))
.getOrElse(Seq.empty)
depMgmts <- xmlDepMgmts.toList.traverseU(dependency)
@ -173,43 +156,94 @@ object Pom {
.map(mod => if (mod.organization.isEmpty) -\/("Parent organization missing") else \/-(()))
.getOrElse(\/-(()))
xmlProperties = pom.child
xmlProperties = pom.children
.find(_.label == "properties")
.map(_.child.collect{case elem if elem.isElement => elem})
.map(_.children.collect{case elem if elem.isElement => elem})
.getOrElse(Seq.empty)
properties <- xmlProperties.toList.traverseU(property)
xmlProfiles = pom.child
xmlProfiles = pom.children
.find(_.label == "profiles")
.map(_.child.filter(_.label == "profile"))
.map(_.children.filter(_.label == "profile"))
.getOrElse(Seq.empty)
profiles <- xmlProfiles.toList.traverseU(profile)
} yield Project(
projModule.copy(organization = groupId),
version,
deps,
parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))),
depMgmts,
properties.toMap,
profiles,
None,
None
)
}
extraAttrs <- properties
.collectFirst { case ("extraDependencyAttributes", s) => extraAttributes(s) }
.getOrElse(\/-(Map.empty))
def parseDateTime(s: String): Option[Versions.DateTime] =
if (s.length == 14 && s.forall(_.isDigit))
Some(Versions.DateTime(
s.substring(0, 4).toInt,
s.substring(4, 6).toInt,
s.substring(6, 8).toInt,
s.substring(8, 10).toInt,
s.substring(10, 12).toInt,
s.substring(12, 14).toInt
))
else
None
extraAttrsMap = extraAttrs.map {
case (mod, ver) =>
(mod.copy(attributes = Map.empty), ver) -> mod.attributes
}.toMap
} yield {
val description = pom.children
.find(_.label == "description")
.map(_.textContent)
.getOrElse("")
val homePage = pom.children
.find(_.label == "url")
.map(_.textContent)
.getOrElse("")
val licenses = pom.children
.find(_.label == "licenses")
.toSeq
.flatMap(_.children)
.filter(_.label == "license")
.flatMap { n =>
text(n, "name", "License name").toOption.map { name =>
(name, text(n, "url", "License URL").toOption)
}.toSeq
}
val developers = pom.children
.find(_.label == "developers")
.toSeq
.flatMap(_.children)
.filter(_.label == "developer")
.map { n =>
for {
id <- text(n, "id", "Developer ID")
name <- text(n, "name", "Developer name")
url <- text(n, "url", "Developer URL")
} yield Info.Developer(id, name, url)
}
.collect {
case \/-(d) => d
}
Project(
projModule.copy(organization = groupId),
version,
deps.map {
case (config, dep0) =>
val dep = extraAttrsMap.get(dep0.moduleVersion).fold(dep0)(attrs =>
dep0.copy(module = dep0.module.copy(attributes = attrs))
)
config -> dep
},
Map.empty,
parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))),
depMgmts,
properties,
profiles,
None,
None,
Nil,
Info(
description,
homePage,
licenses,
developers,
None
)
)
}
}
def versions(node: Node): String \/ Versions = {
import Scalaz._
@ -218,7 +252,7 @@ object Pom {
organization <- text(node, "groupId", "Organization") // Ignored
name <- text(node, "artifactId", "Name") // Ignored
xmlVersioning <- node.child
xmlVersioning <- node.children
.find(_.label == "versioning")
.toRightDisjunction("Versioning info not found in metadata")
@ -227,9 +261,9 @@ object Pom {
release = text(xmlVersioning, "release", "Release version")
.getOrElse("")
versionsOpt = xmlVersioning.child
versionsOpt = xmlVersioning.children
.find(_.label == "versions")
.map(_.child.filter(_.label == "version").flatMap(_.child.collectFirst{case Text(t) => t}))
.map(_.children.filter(_.label == "version").flatMap(_.children.collectFirst{case Text(t) => t}))
lastUpdatedOpt = text(xmlVersioning, "lastUpdated", "Last update date and time")
.toOption
@ -269,7 +303,7 @@ object Pom {
name <- text(node, "artifactId", "Name")
version = readVersion(node)
xmlVersioning <- node.child
xmlVersioning <- node.children
.find(_.label == "versioning")
.toRightDisjunction("Versioning info not found in metadata")
@ -278,15 +312,15 @@ object Pom {
release = text(xmlVersioning, "release", "Release version")
.getOrElse("")
versionsOpt = xmlVersioning.child
versionsOpt = xmlVersioning.children
.find(_.label == "versions")
.map(_.child.filter(_.label == "version").flatMap(_.child.collectFirst{case Text(t) => t}))
.map(_.children.filter(_.label == "version").flatMap(_.children.collectFirst{case Text(t) => t}))
lastUpdatedOpt = text(xmlVersioning, "lastUpdated", "Last update date and time")
.toOption
.flatMap(parseDateTime)
xmlSnapshotOpt = xmlVersioning.child
xmlSnapshotOpt = xmlVersioning.children
.find(_.label == "snapshot")
timestamp = xmlSnapshotOpt
@ -314,15 +348,15 @@ object Pom {
case "false" => false
}
xmlSnapshotVersions = xmlVersioning.child
xmlSnapshotVersions = xmlVersioning.children
.find(_.label == "snapshotVersions")
.map(_.child.filter(_.label == "snapshotVersion"))
.map(_.children.filter(_.label == "snapshotVersion"))
.getOrElse(Seq.empty)
snapshotVersions <- xmlSnapshotVersions
.toList
.traverseU(snapshotVersion)
} yield SnapshotVersioning(
Module(organization, name),
Module(organization, name, Map.empty),
version,
latest,
release,
@ -333,4 +367,69 @@ object Pom {
snapshotVersions
)
}
val extraAttributeSeparator = ":#@#:"
val extraAttributePrefix = "+"
val extraAttributeOrg = "organisation"
val extraAttributeName = "module"
val extraAttributeVersion = "revision"
val extraAttributeBase = Set(
extraAttributeOrg,
extraAttributeName,
extraAttributeVersion,
"branch"
)
val extraAttributeDropPrefix = "e:"
def extraAttribute(s: String): String \/ (Module, String) = {
// vaguely does the same as:
// https://github.com/apache/ant-ivy/blob/2.2.0/src/java/org/apache/ivy/core/module/id/ModuleRevisionId.java#L291
// dropping the attributes with a value of NULL here...
val rawParts = s.split(extraAttributeSeparator).toSeq
val partsOrError =
if (rawParts.length % 2 == 0) {
val malformed = rawParts.filter(!_.startsWith(extraAttributePrefix))
if (malformed.isEmpty)
\/-(rawParts.map(_.drop(extraAttributePrefix.length)))
else
-\/(s"Malformed attributes ${malformed.map("'"+_+"'").mkString(", ")} in extra attributes '$s'")
} else
-\/(s"Malformed extra attributes '$s'")
def attrFrom(attrs: Map[String, String], name: String): String \/ String =
\/.fromEither(
attrs.get(name)
.toRight(s"$name not found in extra attributes '$s'")
)
for {
parts <- partsOrError
attrs = parts.grouped(2).collect {
case Seq(k, v) if v != "NULL" =>
k.stripPrefix(extraAttributeDropPrefix) -> v
}.toMap
org <- attrFrom(attrs, extraAttributeOrg)
name <- attrFrom(attrs, extraAttributeName)
version <- attrFrom(attrs, extraAttributeVersion)
remainingAttrs = attrs.filterKeys(!extraAttributeBase(_))
} yield (Module(org, name, remainingAttrs.toVector.toMap), version)
}
def extraAttributes(s: String): String \/ Seq[(Module, String)] = {
val lines = s.split('\n').toSeq.map(_.trim).filter(_.nonEmpty)
lines.foldLeft[String \/ Seq[(Module, String)]](\/-(Vector.empty)) {
case (acc, line) =>
for {
modVers <- acc
modVer <- extraAttribute(line)
} yield modVers :+ modVer
}
}
}

View File

@ -9,8 +9,8 @@ package object coursier {
def apply(
module: Module,
version: String,
// Substituted by Resolver with its own default scope (compile)
scope: Scope = Scope.Other(""),
// Substituted by Resolver with its own default configuration (compile)
configuration: String = "",
attributes: Attributes = Attributes(),
exclusions: Set[(String, String)] = Set.empty,
optional: Boolean = false
@ -18,9 +18,9 @@ package object coursier {
core.Dependency(
module,
version,
scope,
attributes,
configuration,
exclusions,
attributes,
optional
)
}
@ -37,19 +37,20 @@ package object coursier {
type Project = core.Project
val Project = core.Project
type Info = core.Info
val Info = core.Info
type Profile = core.Profile
val Profile = core.Profile
type Module = core.Module
object Module {
def apply(organization: String, name: String): Module =
core.Module(organization, name)
def apply(organization: String, name: String, attributes: Map[String, String] = Map.empty): Module =
core.Module(organization, name, attributes)
}
type ModuleVersion = (core.Module, String)
type Scope = core.Scope
val Scope = core.Scope
type Repository = core.Repository
val Repository = core.Repository

View File

@ -1,14 +1,33 @@
package coursier.util
import coursier.core.Versions
import scalaz.{\/-, -\/, \/, Scalaz}
object Xml {
/** A representation of an XML node/document, with different implementations on JVM and JS */
trait Node {
def label: String
def child: Seq[Node]
/** Namespace / key / value */
def attributes: Seq[(String, String, String)]
def children: Seq[Node]
def isText: Boolean
def textContent: String
def isElement: Boolean
def attributesFromNamespace(namespace: String): Seq[(String, String)] =
attributes.collect {
case (`namespace`, k, v) =>
k -> v
}
lazy val attributesMap = attributes.map { case (_, k, v) => k -> v }.toMap
def attribute(name: String): String \/ String =
attributesMap.get(name) match {
case None => -\/(s"Missing attribute $name")
case Some(value) => \/-(value)
}
}
object Node {
@ -16,10 +35,39 @@ object Xml {
new Node {
val isText = false
val isElement = false
val child = Nil
val children = Nil
val label = ""
val attributes = Nil
val textContent = ""
}
}
object Text {
def unapply(n: Node): Option[String] =
if (n.isText) Some(n.textContent)
else None
}
def text(elem: Node, label: String, description: String) = {
import Scalaz.ToOptionOpsFromOption
elem.children
.find(_.label == label)
.flatMap(_.children.collectFirst{case Text(t) => t})
.toRightDisjunction(s"$description not found")
}
def parseDateTime(s: String): Option[Versions.DateTime] =
if (s.length == 14 && s.forall(_.isDigit))
Some(Versions.DateTime(
s.substring(0, 4).toInt,
s.substring(4, 6).toInt,
s.substring(6, 8).toInt,
s.substring(8, 10).toInt,
s.substring(10, 12).toInt,
s.substring(12, 14).toInt
))
else
None
}

187
doc/README.md Normal file
View File

@ -0,0 +1,187 @@
# Coursier
*Pure Scala Artifact Fetching*
A pure Scala substitute for [Aether](http://www.eclipse.org/aether/)
[![Build Status](https://travis-ci.org/alexarchambault/coursier.svg?branch=master)](https://travis-ci.org/alexarchambault/coursier)
[![Build status (Windows)](https://ci.appveyor.com/api/projects/status/trtum5b7washfbj9?svg=true)](https://ci.appveyor.com/project/alexarchambault/coursier)
[![Join the chat at https://gitter.im/alexarchambault/coursier](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/alexarchambault/coursier?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![Maven Central](https://img.shields.io/maven-central/v/com.github.alexarchambault/coursier_2.11.svg)](https://maven-badges.herokuapp.com/maven-central/com.github.alexarchambault/coursier_2.11)
*coursier* is a dependency resolver / fetcher *à la* Maven / Ivy, entirely
rewritten from scratch in Scala. It aims at being fast and easy to embed
in other contexts. Its very core (`core` module) aims at being
extremely pure, and should be approached thinking algebraically.
The `files` module handles caching of the metadata and artifacts themselves,
and is less so pure than the `core` module, in the sense that it happily
does IO as a side-effect (although it naturally favors immutability for all
that's kept in memory).
It handles fancy Maven features like
* [POM inheritance](http://books.sonatype.com/mvnref-book/reference/pom-relationships-sect-project-relationships.html#pom-relationships-sect-project-inheritance),
* [dependency management](http://books.sonatype.com/mvnex-book/reference/optimizing-sect-dependencies.html),
* [import scope](https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Importing_Dependencies),
* [properties](http://books.sonatype.com/mvnref-book/reference/resource-filtering-sect-properties.html),
* etc.
It happily resolves dependencies involving modules from the Hadoop ecosystem (Spark, Flink, etc.), that
make a heavy use of these.
It can be used either from the command-line, via its API, or from the browser.
It downloads the metadata or the artifacts in parallel (usually, 6 parallel
downloads).
## Command-line
Download and run its laucher with
```
$ curl -L -o coursier https://git.io/vBSmI && chmod +x coursier && ./coursier --help
```
Note that the launcher itself weights only 8 kB and can be easily
embedded as is in other projects.
The first time it is run, it will download the artifacts required to launch
coursier. You'll be fine the next times :-).
The cache of this default launcher defaults to a directory named `.coursier`,
in the same directory as the launcher. This can be changed by manually adjusting
the `COURSIER_CACHE` variable in the first lines of the launcher.
```
$ ./coursier --help
```
lists the available coursier commands. The most notable ones are `launch`,
`fetch`, and `classpath`. Type
```
$ ./coursier command --help
```
to get a description of the various options the command `command` (replace with one
of the above command) accepts.
### launch
The `launch` command fetches a set of Maven coordinates it is given, along
with their transitive dependencies, then launches the "main `main` class" from
it if it can find one (typically from the manifest of the first coordinates).
The main class to launch can also be manually specified with the `-M` option.
For example, it can launch:
* [Ammonite](https://github.com/lihaoyi/Ammonite) (enhanced Scala REPL),
```
$ ./coursier launch com.lihaoyi:ammonite-repl_2.11.7:0.5.0
```
along with the REPLs of various JVM languages like
* Frege,
```
$ ./coursier launch -r central -r https://oss.sonatype.org/content/groups/public \
org.frege-lang:frege-repl-core:1.3 -M frege.repl.FregeRepl
```
* clojure,
```
$ ./coursier launch org.clojure:clojure:1.7.0 -M clojure.main
```
* jruby,
```
$ wget https://raw.githubusercontent.com/jruby/jruby/master/bin/jirb && \
./coursier launch org.jruby:jruby:9.0.4.0 -M org.jruby.Main -- -- jirb
```
* jython,
```
$ ./coursier launch org.python:jython-standalone:2.7.0 -M org.python.util.jython
```
* Groovy,
```
$ ./coursier launch org.codehaus.groovy:groovy-groovysh:2.4.5 -M org.codehaus.groovy.tools.shell.Main \
commons-cli:commons-cli:1.3.1
```
etc.
and various programs, like
* Proguard and its utility Retrace,
```
$ ./coursier launch net.sf.proguard:proguard-base:5.2.1 -M proguard.ProGuard
$ ./coursier launch net.sf.proguard:proguard-retrace:5.2.1 -M proguard.retrace.ReTrace
```
### fetch
The `fetch` command simply fetches a set of dependencies, along with their
transitive dependencies, then prints the local paths of all their artefacts.
Example
```
$ ./coursier fetch org.apache.spark:spark-sql_2.11:1.5.2
...
/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/io/dropwizard/metrics/metrics-jvm/3.1.2/metrics-jvm-3.1.2.jar
/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/javax/servlet/javax.servlet-api/3.0.1/javax.servlet-api-3.0.1.jar
/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/javax/inject/javax.inject/1/javax.inject-1.jar
...
```
### classpath
The `classpath` command transitively fetches a set of dependencies like
`fetch` does, then prints a classpath that can be handed over directly
to `java`, like
```
$ java -cp "$(./coursier classpath com.lihaoyi:ammonite-repl_2.11.7:0.5.0 | tail -n1)" ammonite.repl.Repl
Loading...
Welcome to the Ammonite Repl 0.5.0
(Scala 2.11.7 Java 1.8.0_60)
@
```
## API
This [gist](https://gist.github.com/larsrh/42da43aa74dc4e78aa59) by [Lars Hupel](https://github.com/larsrh/)
illustrates how the API of coursier can be used to get transitives dependencies
and fetch the corresponding artefacts.
More explanations to come :-)
## Scala JS demo
*coursier* is also compiled to Scala JS, and can be tested in the browser via its
[demo](http://alexarchambault.github.io/coursier/#demo).
# To do / missing
- Snapshots metadata / artifacts, once in cache, are not automatically
updated for now. [#41](https://github.com/alexarchambault/coursier/issues/41)
- File locking could be better (none for metadata, no re-attempt if file locked elsewhere for artifacts) [#71](https://github.com/alexarchambault/coursier/issues/71)
- Handle "configurations" like Ivy does, instead of just the standard
(hard-coded) Maven "scopes" [#8](https://github.com/alexarchambault/coursier/issues/8)
- SBT plugin [#52](https://github.com/alexarchambault/coursier/issues/52),
requires Ivy-like configurations [#8](https://github.com/alexarchambault/coursier/issues/8)
See the list of [issues](https://github.com/alexarchambault/coursier/issues).
# Contributors
- Your name here :-)
Don't hesitate to pick an issue to contribute, and / or ask for help for how to proceed
on the [Gitter channel](https://gitter.im/alexarchambault/coursier).
# Projects using coursier
- [Lars Hupel](https://github.com/larsrh/)'s [libisabelle](https://github.com/larsrh/libisabelle) fetches
some of its requirements via coursier,
- [jupyter-scala](https://github.com/alexarchambault/jupyter-scala) should soon allow
to add dependencies in its sessions with coursier (initial motivation for writing coursier),
- Your project here :-)
Released under the Apache license, v2.

View File

@ -1,26 +0,0 @@
package coursier
import scalaz.concurrent.Task
object Fetch {
implicit def default(
repositories: Seq[core.Repository]
): ResolutionProcess.Fetch[Task] =
apply(repositories, Platform.artifact)
def apply(
repositories: Seq[core.Repository],
fetch: Repository.Fetch[Task]
): ResolutionProcess.Fetch[Task] = {
modVers => Task.gatherUnordered(
modVers.map { case (module, version) =>
Repository.find(repositories, module, version, fetch)
.run
.map((module, version) -> _)
}
)
}
}

View File

@ -8,7 +8,6 @@ import scala.scalajs.js
import js.Dynamic.{ global => g }
import scala.scalajs.js.timers._
import scalaz.concurrent.Task
import scalaz.{ -\/, \/-, EitherT }
object Platform {
@ -75,7 +74,7 @@ object Platform {
p.future
}
val artifact: Repository.Fetch[Task] = { artifact =>
val artifact: Fetch.Content[Task] = { artifact =>
EitherT(
Task { implicit ec =>
get(artifact.url)
@ -87,13 +86,18 @@ object Platform {
)
}
implicit def fetch(
repositories: Seq[core.Repository]
): Fetch.Metadata[Task] =
Fetch(repositories, Platform.artifact)
trait Logger {
def fetching(url: String): Unit
def fetched(url: String): Unit
def other(url: String, msg: String): Unit
}
def artifactWithLogger(logger: Logger): Repository.Fetch[Task] = { artifact =>
def artifactWithLogger(logger: Logger): Fetch.Content[Task] = { artifact =>
EitherT(
Task { implicit ec =>
Future(logger.fetching(artifact.url))

View File

@ -0,0 +1,52 @@
package coursier
import scala.concurrent.{ ExecutionContext, Future }
import scalaz.{ Nondeterminism, Reducer }
/**
* Minimal Future-based Task.
*
* Likely to be flawed and/or sub-optimal, but does the job.
*/
trait Task[T] { self =>
def map[U](f: T => U): Task[U] =
new Task[U] {
def runF(implicit ec: ExecutionContext) = self.runF.map(f)
}
def flatMap[U](f: T => Task[U]): Task[U] =
new Task[U] {
def runF(implicit ec: ExecutionContext) = self.runF.flatMap(f(_).runF)
}
def runF(implicit ec: ExecutionContext): Future[T]
}
object Task {
def now[A](a: A): Task[A] =
new Task[A] {
def runF(implicit ec: ExecutionContext) = Future.successful(a)
}
def apply[A](f: ExecutionContext => Future[A]): Task[A] =
new Task[A] {
def runF(implicit ec: ExecutionContext) = f(ec)
}
def gatherUnordered[T](tasks: Seq[Task[T]], exceptionCancels: Boolean = false): Task[Seq[T]] =
new Task[Seq[T]] {
def runF(implicit ec: ExecutionContext) = Future.traverse(tasks)(_.runF)
}
implicit val taskMonad: Nondeterminism[Task] =
new Nondeterminism[Task] {
def point[A](a: => A): Task[A] = Task.now(a)
def bind[A,B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f)
override def reduceUnordered[A, M](fs: Seq[Task[A]])(implicit R: Reducer[A, M]): Task[M] =
Task { implicit ec =>
val f = Future.sequence(fs.map(_.runF))
f.map { l =>
(R.zero /: l)(R.snoc)
}
}
def chooseAny[A](head: Task[A], tail: Seq[Task[A]]): Task[(A, Seq[Task[A]])] =
???
}
}

View File

@ -1,42 +0,0 @@
package scalaz
import scala.concurrent.{ ExecutionContext, Future }
/** Minimal Future-based Task */
package object concurrent {
trait Task[T] { self =>
def map[U](f: T => U): Task[U] =
new Task[U] {
def runF(implicit ec: ExecutionContext) = self.runF.map(f)
}
def flatMap[U](f: T => Task[U]): Task[U] =
new Task[U] {
def runF(implicit ec: ExecutionContext) = self.runF.flatMap(f(_).runF)
}
def runF(implicit ec: ExecutionContext): Future[T]
}
object Task {
def now[A](a: A): Task[A] =
new Task[A] {
def runF(implicit ec: ExecutionContext) = Future.successful(a)
}
def apply[A](f: ExecutionContext => Future[A]): Task[A] =
new Task[A] {
def runF(implicit ec: ExecutionContext) = f(ec)
}
def gatherUnordered[T](tasks: Seq[Task[T]], exceptionCancels: Boolean = false): Task[Seq[T]] =
new Task[Seq[T]] {
def runF(implicit ec: ExecutionContext) = Future.traverse(tasks)(_.runF)
}
implicit val taskMonad: Monad[Task] =
new Monad[Task] {
def point[A](a: => A): Task[A] = Task.now(a)
def bind[A,B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f)
}
}
}

View File

@ -1,134 +0,0 @@
package coursier
import java.io.{ File, PrintWriter }
import scala.io.Source
object Cache {
def mavenRepository(lines: Seq[String]): Option[MavenRepository] = {
def isMaven =
lines
.find(_.startsWith("maven:"))
.map(_.stripPrefix("maven:").trim)
.toSeq
.contains("true")
def ivyLike =
lines
.find(_.startsWith("ivy-like:"))
.map(_.stripPrefix("ivy-like:").trim)
.toSeq
.contains("true")
def base =
lines
.find(_.startsWith("base:"))
.map(_.stripPrefix("base:").trim)
.filter(_.nonEmpty)
if (isMaven)
base.map(MavenRepository(_, ivyLike = ivyLike))
else
None
}
lazy val default = Cache(new File(sys.props("user.home") + "/.coursier/cache"))
}
case class Cache(cache: File) {
import Cache._
lazy val repoDir = new File(cache, "repositories")
lazy val fileBase = new File(cache, "cache")
lazy val defaultFile = new File(repoDir, ".default")
def add(id: String, base: String, ivyLike: Boolean): Unit = {
repoDir.mkdirs()
val f = new File(repoDir, id)
val w = new PrintWriter(f)
try w.println((Seq("maven: true", s"base: $base") ++ (if (ivyLike) Seq("ivy-like: true") else Nil)).mkString("\n"))
finally w.close()
}
def addCentral(): Unit =
add("central", "https://repo1.maven.org/maven2/", ivyLike = false)
def addIvy2Local(): Unit =
add("ivy2local", new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, ivyLike = true)
def init(
ifEmpty: Boolean = true,
verbose: Boolean = false
): Unit =
if (!ifEmpty || !repoDir.exists() || !fileBase.exists()) {
if (verbose)
Console.err.println(s"Initializing $cache")
repoDir.mkdirs()
fileBase.mkdirs()
addCentral()
addIvy2Local()
setDefault("ivy2local", "central")
}
def setDefault(ids: String*): Unit = {
defaultFile.getParentFile.mkdirs()
val w = new PrintWriter(defaultFile)
try w.println(ids.mkString("\n"))
finally w.close()
}
def list(): Seq[(String, MavenRepository, (String, File))] =
Option(repoDir.listFiles())
.map(_.toSeq)
.getOrElse(Nil)
.filter(f => f.isFile && !f.getName.startsWith("."))
.flatMap { f =>
val name = f.getName
val lines = Source.fromFile(f).getLines().toList
mavenRepository(lines).map(repo =>
(name, repo, (repo.root, new File(fileBase, name)))
)
}
def map(): Map[String, (MavenRepository, (String, File))] =
list()
.map{case (id, repo, fileCache) => id -> (repo, fileCache) }
.toMap
def repositories(): Seq[MavenRepository] =
list().map(_._2)
def repositoryMap(): Map[String, MavenRepository] =
list()
.map{case (id, repo, _) => id -> repo}
.toMap
def fileCaches(): Seq[(String, File)] =
list().map(_._3)
def default(withNotFound: Boolean = false): Seq[String] =
if (defaultFile.exists()) {
val default0 =
Source.fromFile(defaultFile)
.getLines()
.map(_.trim)
.filter(_.nonEmpty)
.toList
val found = list()
.map(_._1)
.toSet
default0
.filter(found)
} else
Nil
def files(): Files =
new Files(list().map{case (_, _, matching) => matching }, () => ???)
}

View File

@ -1,49 +0,0 @@
package coursier
import scalaz.\/
import scalaz.concurrent.Task
sealed trait CachePolicy {
def apply[T](
tryRemote: T => Boolean )(
local: => Task[T] )(
remote: Option[T] => Task[T]
): Task[T]
}
object CachePolicy {
def saving[E,T](
remote: => Task[E \/ T] )(
save: T => Task[Unit]
): Task[E \/ T] = {
for {
res <- remote
_ <- res.fold(_ => Task.now(()), t => save(t))
} yield res
}
case object Default extends CachePolicy {
def apply[T](
tryRemote: T => Boolean )(
local: => Task[T] )(
remote: Option[T] => Task[T]
): Task[T] =
local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res))
}
case object LocalOnly extends CachePolicy {
def apply[T](
tryRemote: T => Boolean )(
local: => Task[T] )(
remote: Option[T] => Task[T]
): Task[T] =
local
}
case object ForceDownload extends CachePolicy {
def apply[T](
tryRemote: T => Boolean )(
local: => Task[T] )(
remote: Option[T] => Task[T]
): Task[T] =
remote(None)
}
}

View File

@ -1,29 +0,0 @@
package coursier
import scalaz.concurrent.Task
object Fetch {
implicit def default(
repositories: Seq[core.Repository]
): ResolutionProcess.Fetch[Task] =
apply(repositories, Platform.artifact)
def apply(
repositories: Seq[core.Repository],
fetch: Repository.Fetch[Task],
extra: Repository.Fetch[Task]*
): ResolutionProcess.Fetch[Task] = {
modVers => Task.gatherUnordered(
modVers.map { case (module, version) =>
def get(fetch: Repository.Fetch[Task]) =
Repository.find(repositories, module, version, fetch)
(get(fetch) /: extra)(_ orElse get(_))
.run
.map((module, version) -> _)
}
)
}
}

View File

@ -1,340 +0,0 @@
package coursier
import java.net.URL
import java.nio.channels.{ OverlappingFileLockException, FileLock }
import java.security.MessageDigest
import java.util.concurrent.{ Executors, ExecutorService }
import scala.annotation.tailrec
import scalaz._
import scalaz.concurrent.{ Task, Strategy }
import java.io._
case class Files(
cache: Seq[(String, File)],
tmp: () => File,
concurrentDownloadCount: Int = Files.defaultConcurrentDownloadCount
) {
lazy val defaultPool =
Executors.newFixedThreadPool(concurrentDownloadCount, Strategy.DefaultDaemonThreadFactory)
def withLocal(artifact: Artifact): Artifact = {
def local(url: String) =
if (url.startsWith("file:///"))
url.stripPrefix("file://")
else if (url.startsWith("file:/"))
url.stripPrefix("file:")
else
cache.find { case (base, _) => url.startsWith(base) } match {
case None =>
// FIXME Means we were handed an artifact from repositories other than the known ones
println(cache.mkString("\n"))
println(url)
???
case Some((base, cacheDir)) =>
cacheDir + "/" + url.stripPrefix(base)
}
if (artifact.extra.contains("local"))
artifact
else
artifact.copy(extra = artifact.extra + ("local" ->
artifact.copy(
url = local(artifact.url),
checksumUrls = artifact.checksumUrls
.mapValues(local)
.toVector
.toMap,
extra = Map.empty
)
))
}
def download(
artifact: Artifact,
withChecksums: Boolean = true,
logger: Option[Files.Logger] = None
)(implicit
cachePolicy: CachePolicy,
pool: ExecutorService = defaultPool
): Task[Seq[((File, String), FileError \/ Unit)]] = {
val artifact0 = withLocal(artifact)
.extra
.getOrElse("local", artifact)
val pairs =
Seq(artifact0.url -> artifact.url) ++ {
if (withChecksums)
(artifact0.checksumUrls.keySet intersect artifact.checksumUrls.keySet)
.toList
.map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
else
Nil
}
def locally(file: File, url: String) =
Task {
if (file.exists()) {
logger.foreach(_.foundLocally(url, file))
\/-(file)
} else
-\/(FileError.NotFound(file.toString): FileError)
}
// FIXME Things can go wrong here and are not properly handled,
// e.g. what if the connection gets closed during the transfer?
// (partial file on disk?)
def remote(file: File, url: String) =
Task {
try {
logger.foreach(_.downloadingArtifact(url))
val conn = new URL(url).openConnection() // FIXME Should this be closed?
// Dummy user-agent instead of the default "Java/...",
// so that we are not returned incomplete/erroneous metadata
// (Maven 2 compatibility? - happens for snapshot versioning metadata,
// this is SO FUCKING CRAZY)
conn.setRequestProperty("User-Agent", "")
for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L))
logger.foreach(_.downloadLength(url, len))
val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize)
val result =
try {
file.getParentFile.mkdirs()
val out = new FileOutputStream(file)
try {
var lock: FileLock = null
try {
lock = out.getChannel.tryLock()
if (lock == null)
-\/(FileError.Locked(file.toString))
else {
val b = Array.fill[Byte](Files.bufferSize)(0)
@tailrec
def helper(count: Long): Unit = {
val read = in.read(b)
if (read >= 0) {
out.write(b, 0, read)
logger.foreach(_.downloadProgress(url, count + read))
helper(count + read)
}
}
helper(0L)
\/-(file)
}
}
catch {
case e: OverlappingFileLockException =>
-\/(FileError.Locked(file.toString))
}
finally if (lock != null) lock.release()
} finally out.close()
} finally in.close()
for (lastModified <- Option(conn.getLastModified).filter(_ > 0L))
file.setLastModified(lastModified)
logger.foreach(_.downloadedArtifact(url, success = true))
result
}
catch { case e: Exception =>
logger.foreach(_.downloadedArtifact(url, success = false))
-\/(FileError.DownloadError(e.getMessage))
}
}
val tasks =
for ((f, url) <- pairs) yield {
val file = new File(f)
if (url != ("file:" + f) && url != ("file://" + f)) {
assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url")
cachePolicy[FileError \/ File](
_.isLeft )(
locally(file, url) )(
_ => remote(file, url)
).map(e => (file, url) -> e.map(_ => ()))
} else
Task {
(file, url) -> {
if (file.exists())
\/-(())
else
-\/(FileError.NotFound(file.toString))
}
}
}
Nondeterminism[Task].gather(tasks)
}
def validateChecksum(
artifact: Artifact,
sumType: String
)(implicit
pool: ExecutorService = defaultPool
): Task[FileError \/ Unit] = {
val artifact0 = withLocal(artifact)
.extra
.getOrElse("local", artifact)
artifact0.checksumUrls.get(sumType) match {
case Some(sumFile) =>
Task {
val sum = scala.io.Source.fromFile(sumFile)
.getLines()
.toStream
.headOption
.mkString
.takeWhile(!_.isSpaceChar)
val md = MessageDigest.getInstance(sumType)
val is = new FileInputStream(new File(artifact0.url))
try Files.withContent(is, md.update(_, 0, _))
finally is.close()
val digest = md.digest()
val calculatedSum = f"${BigInt(1, digest)}%040x"
if (sum == calculatedSum)
\/-(())
else
-\/(FileError.WrongChecksum(sumType, calculatedSum, sum, artifact0.url, sumFile))
}
case None =>
Task.now(-\/(FileError.ChecksumNotFound(sumType, artifact0.url)))
}
}
def file(
artifact: Artifact,
checksum: Option[String] = Some("SHA-1"),
logger: Option[Files.Logger] = None
)(implicit
cachePolicy: CachePolicy,
pool: ExecutorService = defaultPool
): EitherT[Task, FileError, File] =
EitherT {
val res = download(artifact, withChecksums = checksum.nonEmpty, logger = logger).map {
results =>
val ((f, _), res) = results.head
res.map(_ => f)
}
checksum.fold(res) { sumType =>
res.flatMap {
case err @ -\/(_) => Task.now(err)
case \/-(f) =>
validateChecksum(artifact, sumType)
.map(_.map(_ => f))
}
}
}
def fetch(
checksum: Option[String] = Some("SHA-1"),
logger: Option[Files.Logger] = None
)(implicit
cachePolicy: CachePolicy,
pool: ExecutorService = defaultPool
): Repository.Fetch[Task] = {
artifact =>
file(artifact, checksum = checksum, logger = logger)(cachePolicy).leftMap(_.message).map { f =>
// FIXME Catch error here?
scala.io.Source.fromFile(f)("UTF-8").mkString
}
}
}
object Files {
lazy val ivy2Local = MavenRepository(
new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString,
ivyLike = true
)
val defaultConcurrentDownloadCount = 6
trait Logger {
def foundLocally(url: String, f: File): Unit = {}
def downloadingArtifact(url: String): Unit = {}
def downloadLength(url: String, length: Long): Unit = {}
def downloadProgress(url: String, downloaded: Long): Unit = {}
def downloadedArtifact(url: String, success: Boolean): Unit = {}
}
var bufferSize = 1024*1024
def readFullySync(is: InputStream) = {
val buffer = new ByteArrayOutputStream()
val data = Array.ofDim[Byte](16384)
var nRead = is.read(data, 0, data.length)
while (nRead != -1) {
buffer.write(data, 0, nRead)
nRead = is.read(data, 0, data.length)
}
buffer.flush()
buffer.toByteArray
}
def readFully(is: => InputStream) =
Task {
\/.fromTryCatchNonFatal {
val is0 = is
val b =
try readFullySync(is0)
finally is0.close()
new String(b, "UTF-8")
} .leftMap(_.getMessage)
}
def withContent(is: InputStream, f: (Array[Byte], Int) => Unit): Unit = {
val data = Array.ofDim[Byte](16384)
var nRead = is.read(data, 0, data.length)
while (nRead != -1) {
f(data, nRead)
nRead = is.read(data, 0, data.length)
}
}
}
sealed trait FileError {
def message: String
}
object FileError {
case class DownloadError(message0: String) extends FileError {
def message = s"Download error: $message0"
}
case class NotFound(file: String) extends FileError {
def message = s"$file: not found"
}
case class Locked(file: String) extends FileError {
def message = s"$file: locked"
}
case class ChecksumNotFound(sumType: String, file: String) extends FileError {
def message = s"$file: $sumType checksum not found"
}
case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError {
def message = s"$file: $sumType checksum validation failed"
}
}

View File

@ -0,0 +1,52 @@
package coursier
import java.io.File
import sbt._
import sbt.Keys._
object CoursierPlugin extends AutoPlugin {
override def trigger = allRequirements
override def requires = sbt.plugins.IvyPlugin
object autoImport {
val coursierParallelDownloads = Keys.coursierParallelDownloads
val coursierMaxIterations = Keys.coursierMaxIterations
val coursierChecksums = Keys.coursierChecksums
val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums
val coursierCachePolicy = Keys.coursierCachePolicy
val coursierVerbosity = Keys.coursierVerbosity
val coursierResolvers = Keys.coursierResolvers
val coursierSbtResolvers = Keys.coursierSbtResolvers
val coursierCache = Keys.coursierCache
val coursierProject = Keys.coursierProject
val coursierProjects = Keys.coursierProjects
val coursierPublications = Keys.coursierPublications
val coursierSbtClassifiersModule = Keys.coursierSbtClassifiersModule
}
import autoImport._
override lazy val projectSettings = Seq(
coursierParallelDownloads := 6,
coursierMaxIterations := 50,
coursierChecksums := Seq(Some("SHA-1"), None),
coursierArtifactsChecksums := Seq(None),
coursierCachePolicy := CachePolicy.FetchMissing,
coursierVerbosity := 1,
coursierResolvers <<= Tasks.coursierResolversTask,
coursierSbtResolvers <<= externalResolvers in updateSbtClassifiers,
coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"),
update <<= Tasks.updateTask(withClassifiers = false),
updateClassifiers <<= Tasks.updateTask(withClassifiers = true),
updateSbtClassifiers in Defaults.TaskGlobal <<= Tasks.updateTask(withClassifiers = true, sbtClassifiers = true),
coursierProject <<= Tasks.coursierProjectTask,
coursierProjects <<= Tasks.coursierProjectsTask,
coursierPublications <<= Tasks.coursierPublicationsTask,
coursierSbtClassifiersModule <<= classifiersModule in updateSbtClassifiers
)
}

View File

@ -0,0 +1,140 @@
package coursier
import coursier.ivy.{ IvyXml, IvyRepository }
import sbt.mavenint.SbtPomExtraProperties
import sbt.{ Resolver, CrossVersion, ModuleID }
object FromSbt {
def sbtModuleIdName(
moduleId: ModuleID,
scalaVersion: => String,
scalaBinaryVersion: => String
): String =
sbtCrossVersionName(moduleId.name, moduleId.crossVersion, scalaVersion, scalaBinaryVersion)
def sbtCrossVersionName(
name: String,
crossVersion: CrossVersion,
scalaVersion: => String,
scalaBinaryVersion: => String
): String = crossVersion match {
case CrossVersion.Disabled => name
case f: CrossVersion.Full => name + "_" + f.remapVersion(scalaVersion)
case f: CrossVersion.Binary => name + "_" + f.remapVersion(scalaBinaryVersion)
}
def attributes(attr: Map[String, String]): Map[String, String] =
attr.map { case (k, v) =>
k.stripPrefix("e:") -> v
}.filter { case (k, _) =>
!k.startsWith(SbtPomExtraProperties.POM_INFO_KEY_PREFIX)
}
def dependencies(
module: ModuleID,
scalaVersion: String,
scalaBinaryVersion: String
): Seq[(String, Dependency)] = {
// TODO Warn about unsupported properties in `module`
val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion)
val dep = Dependency(
Module(module.organization, fullName, FromSbt.attributes(module.extraDependencyAttributes)),
module.revision,
exclusions = module.exclusions.map { rule =>
// FIXME Other `rule` fields are ignored here
(rule.organization, rule.name)
}.toSet
)
val mapping = module.configurations.getOrElse("compile")
val allMappings = IvyXml.mappings(mapping)
val attributes =
if (module.explicitArtifacts.isEmpty)
Seq(Attributes())
else
module.explicitArtifacts.map { a =>
Attributes(`type` = a.extension, classifier = a.classifier.getOrElse(""))
}
for {
(from, to) <- allMappings.toSeq
attr <- attributes
} yield from -> dep.copy(configuration = to, attributes = attr)
}
def project(
projectID: ModuleID,
allDependencies: Seq[ModuleID],
ivyConfigurations: Map[String, Seq[String]],
scalaVersion: String,
scalaBinaryVersion: String
): Project = {
// FIXME Ignored for now - easy to support though
// val sbtDepOverrides = dependencyOverrides.value
// val sbtExclusions = excludeDependencies.value
val deps = allDependencies.flatMap(dependencies(_, scalaVersion, scalaBinaryVersion))
Project(
Module(
projectID.organization,
sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion),
FromSbt.attributes(projectID.extraDependencyAttributes)
),
projectID.revision,
deps,
ivyConfigurations,
None,
Nil,
Nil,
Nil,
None,
None,
Nil,
Info.empty
)
}
def repository(resolver: Resolver, ivyProperties: Map[String, String]): Option[Repository] =
resolver match {
case sbt.MavenRepository(_, root) =>
if (root.startsWith("http://") || root.startsWith("https://")) {
val root0 = if (root.endsWith("/")) root else root + "/"
Some(MavenRepository(root0, sbtAttrStub = true))
} else {
Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it")
None
}
case sbt.FileRepository(_, _, patterns)
if patterns.ivyPatterns.lengthCompare(1) == 0 &&
patterns.ivyPatterns == patterns.artifactPatterns =>
Some(IvyRepository(
"file://" + patterns.ivyPatterns.head,
changing = Some(true),
properties = ivyProperties
))
case sbt.URLRepository(_, patterns)
if patterns.ivyPatterns.lengthCompare(1) == 0 &&
patterns.ivyPatterns == patterns.artifactPatterns =>
Some(IvyRepository(
patterns.ivyPatterns.head,
changing = None,
properties = ivyProperties
))
case other =>
Console.err.println(s"Warning: unrecognized repository ${other.name}, ignoring it")
None
}
}

View File

@ -0,0 +1,27 @@
package coursier
import scalaz.{ -\/, \/-, Monad, EitherT }
case class InterProjectRepository(projects: Seq[Project]) extends Repository {
private val map = projects
.map { proj => proj.moduleVersion -> proj }
.toMap
def find[F[_]](
module: Module,
version: String,
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
): EitherT[F, String, (Artifact.Source, Project)] = {
val res = map.get((module, version)) match {
case Some(proj) =>
\/-((Artifact.Source.empty, proj))
case None =>
-\/("Not found")
}
EitherT(F.point(res))
}
}

View File

@ -0,0 +1,26 @@
package coursier
import java.io.File
import coursier.core.Publication
import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey }
object Keys {
val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads", "")
val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations", "")
val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "")
val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums", "")
val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "")
val coursierVerbosity = SettingKey[Int]("coursier-verbosity", "")
val coursierResolvers = TaskKey[Seq[Resolver]]("coursier-resolvers", "")
val coursierSbtResolvers = TaskKey[Seq[Resolver]]("coursier-sbt-resolvers", "")
val coursierCache = SettingKey[File]("coursier-cache", "")
val coursierProject = TaskKey[Project]("coursier-project", "")
val coursierProjects = TaskKey[Seq[Project]]("coursier-projects", "")
val coursierPublications = TaskKey[Seq[(String, Publication)]]("coursier-publications", "")
val coursierSbtClassifiersModule = TaskKey[GetClassifiersModule]("coursier-sbt-classifiers-module", "")
}

View File

@ -0,0 +1,64 @@
package coursier
import scala.xml.{ Node, PrefixedAttribute }
object MakeIvyXml {
def apply(project: Project): Node = {
val baseInfoAttrs = <x
organisation={project.module.organization}
module={project.module.name}
revision={project.version}
/>.attributes
val infoAttrs = project.module.attributes.foldLeft(baseInfoAttrs) {
case (acc, (k, v)) =>
new PrefixedAttribute("e", k, v, acc)
}
val licenseElems = project.info.licenses.map {
case (name, urlOpt) =>
var n = <license name={name} />
for (url <- urlOpt)
n = n % <x url={url} />.attributes
n
}
val infoElem = {
<info>
{licenseElems}
<description>{project.info.description}</description>
</info>
} % infoAttrs
val confElems = project.configurations.toVector.map {
case (name, extends0) =>
var n = <conf name={name} visibility="public" description="" />
if (extends0.nonEmpty)
n = n % <x extends={extends0.mkString(",")} />.attributes
n
}
val publicationElems = project.publications.map {
case (conf, pub) =>
var n = <artifact name={pub.name} type={pub.`type`} ext={pub.ext} conf={conf} />
if (pub.classifier.nonEmpty)
n = n % <x e:classifier={pub.classifier} />.attributes
n
}
val dependencyElems = project.dependencies.toVector.map {
case (conf, dep) =>
<dependency org={dep.module.organization} name={dep.module.name} rev={dep.version} conf={s"$conf->${dep.configuration}"} />
}
<ivy-module version="2.0" xmlns:e="http://ant.apache.org/ivy/extra">
{infoElem}
<configurations>{confElems}</configurations>
<publications>{publicationElems}</publications>
<dependencies>{dependencyElems}</dependencies>
</ivy-module>
}
}

View File

@ -0,0 +1,40 @@
package coursier
import sbt._
// things from sbt-structure
object Structure {
import Def.Initialize._
def structure(state: State): Load.BuildStructure =
sbt.Project.structure(state)
implicit def `enrich SettingKey`[T](key: SettingKey[T]) = new {
def find(state: State): Option[T] =
key.get(structure(state).data)
def get(state: State): T =
find(state).get
def getOrElse(state: State, default: => T): T =
find(state).getOrElse(default)
}
implicit def `enrich TaskKey`[T](key: TaskKey[T]) = new {
def find(state: State): Option[sbt.Task[T]] =
key.get(structure(state).data)
def get(state: State): sbt.Task[T] =
find(state).get
def forAllProjects(state: State, projects: Seq[ProjectRef]): sbt.Task[Map[ProjectRef, T]] = {
val tasks = projects.flatMap(p => key.in(p).get(structure(state).data).map(_.map(it => (p, it))))
std.TaskExtra.joinTasks(tasks).join.map(_.toMap)
}
def forAllConfigurations(state: State, configurations: Seq[sbt.Configuration]): sbt.Task[Map[sbt.Configuration, T]] = {
val tasks = configurations.flatMap(c => key.in(c).get(structure(state).data).map(_.map(it => (c, it))))
std.TaskExtra.joinTasks(tasks).join.map(_.toMap)
}
}
}

View File

@ -0,0 +1,436 @@
package coursier
import java.io.{ OutputStreamWriter, File }
import java.nio.file.Files
import java.util.concurrent.Executors
import coursier.core.Publication
import coursier.ivy.IvyRepository
import coursier.Keys._
import coursier.Structure._
import org.apache.ivy.core.module.id.ModuleRevisionId
import sbt.{ UpdateReport, Classpaths, Resolver, Def }
import sbt.Configurations.{ Compile, Test }
import sbt.Keys._
import scala.collection.mutable
import scala.collection.JavaConverters._
import scalaz.{ \/-, -\/ }
import scalaz.concurrent.{ Task, Strategy }
object Tasks {
def coursierResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = Def.task {
var resolvers = externalResolvers.value
if (sbtPlugin.value)
resolvers = Seq(
sbtResolver.value,
Classpaths.sbtPluginReleases
) ++ resolvers
resolvers
}
def coursierProjectTask: Def.Initialize[sbt.Task[Project]] =
(
sbt.Keys.state,
sbt.Keys.thisProjectRef
).flatMap { (state, projectRef) =>
// should projectID.configurations be used instead?
val configurations = ivyConfigurations.in(projectRef).get(state)
val allDependenciesTask = allDependencies.in(projectRef).get(state)
for {
allDependencies <- allDependenciesTask
} yield {
FromSbt.project(
projectID.in(projectRef).get(state),
allDependencies,
configurations.map { cfg => cfg.name -> cfg.extendsConfigs.map(_.name) }.toMap,
scalaVersion.in(projectRef).get(state),
scalaBinaryVersion.in(projectRef).get(state)
)
}
}
def coursierProjectsTask: Def.Initialize[sbt.Task[Seq[Project]]] =
sbt.Keys.state.flatMap { state =>
val projects = structure(state).allProjectRefs
coursierProject.forAllProjects(state, projects).map(_.values.toVector)
}
def coursierPublicationsTask: Def.Initialize[sbt.Task[Seq[(String, Publication)]]] =
(
sbt.Keys.state,
sbt.Keys.thisProjectRef,
sbt.Keys.projectID,
sbt.Keys.scalaVersion,
sbt.Keys.scalaBinaryVersion
).map { (state, projectRef, projId, sv, sbv) =>
val packageTasks = Seq(packageBin, packageSrc, packageDoc)
val configs = Seq(Compile, Test)
val sbtArtifacts =
for {
pkgTask <- packageTasks
config <- configs
} yield {
val publish = publishArtifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, false)
if (publish)
Option(artifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, null))
.map(config.name -> _)
else
None
}
sbtArtifacts.collect {
case Some((config, artifact)) =>
val name = FromSbt.sbtCrossVersionName(
artifact.name,
projId.crossVersion,
sv,
sbv
)
val publication = Publication(
name,
artifact.`type`,
artifact.extension,
artifact.classifier.getOrElse("")
)
config -> publication
}
}
// FIXME More things should possibly be put here too (resolvers, etc.)
private case class CacheKey(
resolution: Resolution,
withClassifiers: Boolean,
sbtClassifiers: Boolean
)
private val resolutionsCache = new mutable.HashMap[CacheKey, UpdateReport]
def updateTask(withClassifiers: Boolean, sbtClassifiers: Boolean = false) = Def.task {
// SBT logging should be better than that most of the time...
def errPrintln(s: String): Unit = scala.Console.err.println(s)
def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] =
map.groupBy { case (k, _) => k }.map {
case (k, l) =>
k -> l.map { case (_, v) => v }
}
// let's update only one module at once, for a better output
// Downloads are already parallel, no need to parallelize further anyway
synchronized {
lazy val cm = coursierSbtClassifiersModule.value
val currentProject =
if (sbtClassifiers)
FromSbt.project(
cm.id,
cm.modules,
cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap,
scalaVersion.value,
scalaBinaryVersion.value
)
else {
val proj = coursierProject.value
val publications = coursierPublications.value
proj.copy(publications = publications)
}
val ivySbt0 = ivySbt.value
val ivyCacheManager = ivySbt0.withIvy(streams.value.log)(ivy =>
ivy.getResolutionCacheManager
)
val ivyModule = ModuleRevisionId.newInstance(
currentProject.module.organization,
currentProject.module.name,
currentProject.version,
currentProject.module.attributes.asJava
)
val cacheIvyFile = ivyCacheManager.getResolvedIvyFileInCache(ivyModule)
val cacheIvyPropertiesFile = ivyCacheManager.getResolvedIvyPropertiesInCache(ivyModule)
val projects = coursierProjects.value
val parallelDownloads = coursierParallelDownloads.value
val checksums = coursierChecksums.value
val artifactsChecksums = coursierArtifactsChecksums.value
val maxIterations = coursierMaxIterations.value
val cachePolicy = coursierCachePolicy.value
val cacheDir = coursierCache.value
val resolvers =
if (sbtClassifiers)
coursierSbtResolvers.value
else
coursierResolvers.value
val verbosity = coursierVerbosity.value
val startRes = Resolution(
currentProject.dependencies.map { case (_, dep) => dep }.toSet,
filter = Some(dep => !dep.optional),
forceVersions = projects.map(_.moduleVersion).toMap
)
// required for publish to be fine, later on
def writeIvyFiles() = {
val printer = new scala.xml.PrettyPrinter(80, 2)
val b = new StringBuilder
b ++= """<?xml version="1.0" encoding="UTF-8"?>"""
b += '\n'
b ++= printer.format(MakeIvyXml(currentProject))
cacheIvyFile.getParentFile.mkdirs()
Files.write(cacheIvyFile.toPath, b.result().getBytes("UTF-8"))
// Just writing an empty file here... Are these only used?
cacheIvyPropertiesFile.getParentFile.mkdirs()
Files.write(cacheIvyPropertiesFile.toPath, "".getBytes("UTF-8"))
}
def report = {
if (verbosity >= 1) {
println("InterProjectRepository")
for (p <- projects)
println(s" ${p.module}:${p.version}")
}
val globalPluginsRepo = IvyRepository(
new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString +
"[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]",
withChecksums = false,
withSignatures = false,
withArtifacts = false
)
val interProjectRepo = InterProjectRepository(projects)
val ivyProperties = Map(
"ivy.home" -> s"${sys.props("user.home")}/.ivy2"
) ++ sys.props
val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties))
val caches = Seq(
"http://" -> new File(cacheDir, "http"),
"https://" -> new File(cacheDir, "https")
)
val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory)
def createLogger() = new TermDisplay(
new OutputStreamWriter(System.err),
fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty
)
val resLogger = createLogger()
val fetch = coursier.Fetch(
repositories,
Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool),
Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool)
)
def depsRepr(deps: Seq[(String, Dependency)]) =
deps.map { case (config, dep) =>
s"${dep.module}:${dep.version}:$config->${dep.configuration}"
}.sorted.distinct
def depsRepr0(deps: Seq[Dependency]) =
deps.map { dep =>
s"${dep.module}:${dep.version}:${dep.configuration}"
}.sorted.distinct
if (verbosity >= 1) {
errPrintln(s"Repositories:")
val repositories0 = repositories.map {
case r: IvyRepository => r.copy(properties = Map.empty)
case r: InterProjectRepository => r.copy(projects = Nil)
case r => r
}
for (repo <- repositories0)
errPrintln(s" $repo")
}
if (verbosity >= 0)
errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}")
if (verbosity >= 1)
for (depRepr <- depsRepr(currentProject.dependencies))
errPrintln(s" $depRepr")
resLogger.init()
val res = startRes
.process
.run(fetch, maxIterations)
.attemptRun
.leftMap(ex => throw new Exception(s"Exception during resolution", ex))
.merge
resLogger.stop()
if (!res.isDone)
throw new Exception(s"Maximum number of iteration reached!")
if (verbosity >= 0)
errPrintln("Resolution done")
if (verbosity >= 1)
for (depRepr <- depsRepr0(res.minDependencies.toSeq))
errPrintln(s" $depRepr")
def repr(dep: Dependency) = {
// dep.version can be an interval, whereas the one from project can't
val version = res
.projectCache
.get(dep.moduleVersion)
.map(_._2.version)
.getOrElse(dep.version)
val extra =
if (version == dep.version) ""
else s" ($version for ${dep.version})"
(
Seq(
dep.module.organization,
dep.module.name,
dep.attributes.`type`
) ++
Some(dep.attributes.classifier)
.filter(_.nonEmpty)
.toSeq ++
Seq(
version
)
).mkString(":") + extra
}
if (res.conflicts.nonEmpty) {
// Needs test
println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}")
}
val errors = res.errors
if (errors.nonEmpty) {
println(s"\n${errors.size} error(s):")
for ((dep, errs) <- errors) {
println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}")
}
throw new Exception(s"Encountered ${errors.length} error(s)")
}
val classifiers =
if (withClassifiers)
Some {
if (sbtClassifiers)
cm.classifiers
else
transitiveClassifiers.value
}
else
None
val allArtifacts =
classifiers match {
case None => res.artifacts
case Some(cl) => res.classifiersArtifacts(cl)
}
val artifactsLogger = createLogger()
val artifactFileOrErrorTasks = allArtifacts.toVector.map { a =>
Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _))
}
if (verbosity >= 0)
errPrintln(s"Fetching artifacts")
artifactsLogger.init()
val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match {
case -\/(ex) =>
throw new Exception(s"Error while downloading / verifying artifacts", ex)
case \/-(l) =>
l.toMap
}
artifactsLogger.stop()
if (verbosity >= 0)
errPrintln(s"Fetching artifacts: done")
val configs = {
val configs0 = ivyConfigurations.value.map { config =>
config.name -> config.extendsConfigs.map(_.name)
}.toMap
def allExtends(c: String) = {
// possibly bad complexity
def helper(current: Set[String]): Set[String] = {
val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil))
if ((newSet -- current).nonEmpty)
helper(newSet)
else
newSet
}
helper(Set(c))
}
configs0.map {
case (config, _) =>
config -> allExtends(config)
}
}
def artifactFileOpt(artifact: Artifact) = {
val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded"))
fileOrError match {
case \/-(file) =>
if (file.toString.contains("file:/"))
throw new Exception(s"Wrong path: $file")
Some(file)
case -\/(err) =>
errPrintln(s"${artifact.url}: $err")
None
}
}
val depsByConfig = grouped(currentProject.dependencies)
writeIvyFiles()
ToSbt.updateReport(
depsByConfig,
res,
configs,
classifiers,
artifactFileOpt
)
}
resolutionsCache.getOrElseUpdate(
CacheKey(startRes.copy(filter = None), withClassifiers, sbtClassifiers),
report
)
}
}
}

View File

@ -0,0 +1,179 @@
package coursier
import java.util.GregorianCalendar
import sbt._
object ToSbt {
def moduleId(dependency: Dependency): sbt.ModuleID =
sbt.ModuleID(
dependency.module.organization,
dependency.module.name,
dependency.version,
configurations = Some(dependency.configuration),
extraAttributes = dependency.module.attributes
)
def artifact(module: Module, artifact: Artifact): sbt.Artifact =
sbt.Artifact(
module.name,
artifact.attributes.`type`,
"jar",
Some(artifact.attributes.classifier).filter(_.nonEmpty),
Nil,
Some(url(artifact.url)),
Map.empty
)
def moduleReport(
dependency: Dependency,
dependees: Seq[(Dependency, Project)],
project: Project,
artifacts: Seq[(Artifact, Option[File])]
): sbt.ModuleReport = {
val sbtArtifacts = artifacts.collect {
case (artifact, Some(file)) =>
(ToSbt.artifact(dependency.module, artifact), file)
}
val sbtMissingArtifacts = artifacts.collect {
case (artifact, None) =>
ToSbt.artifact(dependency.module, artifact)
}
val publicationDate = project.info.publication.map { dt =>
new GregorianCalendar(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second).getTime
}
val callers = dependees.map {
case (dependee, dependeeProj) =>
new Caller(
ToSbt.moduleId(dependee),
dependeeProj.configurations.keys.toVector,
dependee.module.attributes ++ dependeeProj.properties,
// FIXME Set better values here
isForceDependency = false,
isChangingDependency = false,
isTransitiveDependency = false,
isDirectlyForceDependency = false
)
}
new sbt.ModuleReport(
module = ToSbt.moduleId(dependency),
artifacts = sbtArtifacts,
missingArtifacts = sbtMissingArtifacts,
status = None,
publicationDate = publicationDate,
resolver = None,
artifactResolver = None,
evicted = false,
evictedData = None,
evictedReason = None,
problem = None,
homepage = Some(project.info.homePage).filter(_.nonEmpty),
extraAttributes = dependency.module.attributes ++ project.properties,
isDefault = None,
branch = None,
configurations = project.configurations.keys.toVector,
licenses = project.info.licenses,
callers = callers
)
}
private def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] =
map.groupBy { case (k, _) => k }.map {
case (k, l) =>
k -> l.map { case (_, v) => v }
}
def moduleReports(
res: Resolution,
classifiersOpt: Option[Seq[String]],
artifactFileOpt: Artifact => Option[File]
) = {
val depArtifacts =
classifiersOpt match {
case None => res.dependencyArtifacts
case Some(cl) => res.dependencyClassifiersArtifacts(cl)
}
val groupedDepArtifacts = grouped(depArtifacts)
val versions = res.dependencies.toVector.map { dep =>
dep.module -> dep.version
}.toMap
def clean(dep: Dependency): Dependency =
dep.copy(configuration = "", exclusions = Set.empty, optional = false)
val reverseDependencies = res.reverseDependencies
.toVector
.map { case (k, v) =>
clean(k) -> v.map(clean)
}
.groupBy { case (k, v) => k }
.mapValues { v =>
v.flatMap {
case (_, l) => l
}
}
.toVector
.toMap
groupedDepArtifacts.map {
case (dep, artifacts) =>
val (_, proj) = res.projectCache(dep.moduleVersion)
// FIXME Likely flaky...
val dependees = reverseDependencies
.getOrElse(clean(dep.copy(version = "")), Vector.empty)
.map { dependee0 =>
val version = versions(dependee0.module)
val dependee = dependee0.copy(version = version)
val (_, dependeeProj) = res.projectCache(dependee.moduleVersion)
(dependee, dependeeProj)
}
ToSbt.moduleReport(
dep,
dependees,
proj,
artifacts.map(a => a -> artifactFileOpt(a))
)
}
}
def updateReport(
configDependencies: Map[String, Seq[Dependency]],
resolution: Resolution,
configs: Map[String, Set[String]],
classifiersOpt: Option[Seq[String]],
artifactFileOpt: Artifact => Option[File]
): sbt.UpdateReport = {
val configReports = configs.map {
case (config, extends0) =>
val configDeps = extends0.flatMap(configDependencies.getOrElse(_, Nil))
val subRes = resolution.subset(configDeps)
val reports = ToSbt.moduleReports(subRes, classifiersOpt, artifactFileOpt)
new ConfigurationReport(
config,
reports.toVector,
Nil,
Nil
)
}
new UpdateReport(
null,
configReports.toVector,
new UpdateStats(-1L, -1L, -1L, cached = false),
Map.empty
)
}
}

View File

@ -3,3 +3,4 @@ addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.5")
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5")
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.1.0")
addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.4.0")

View File

@ -29,21 +29,26 @@ function isMasterOrDevelop() {
# web sub-project doesn't compile in 2.10 (no scalajs-react)
if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then
SBT_COMMANDS="cli/compile"
IS_210=1
SBT_COMMANDS="bootstrap/compile coreJVM/compile coreJS/compile cache/compile web/compile testsJVM/test testsJS/test"
else
SBT_COMMANDS="compile"
IS_210=0
SBT_COMMANDS="compile test"
fi
# Required for ~/.ivy2/local repo tests
~/sbt coreJVM/publish-local
SBT_COMMANDS="$SBT_COMMANDS test"
# TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
PUSH_GHPAGES=0
if isNotPr && publish && isMaster; then
SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish files/publish cli/publish"
SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish cache/publish"
if [ "$IS_210" = 1 ]; then
SBT_COMMANDS="$SBT_COMMANDS plugin/publish"
else
SBT_COMMANDS="$SBT_COMMANDS cli/publish"
fi
fi
if isNotPr && publish && isMasterOrDevelop; then

View File

@ -1,6 +1,6 @@
package coursier.test
import coursier.{ Module, Files }
import coursier.{ Module, Cache }
import utest._
object IvyLocalTests extends TestSuite {
@ -9,8 +9,8 @@ object IvyLocalTests extends TestSuite {
'coursier{
// Assume this module (and the sub-projects it depends on) is published locally
CentralTests.resolutionCheck(
Module("com.github.alexarchambault", "coursier_2.11"), "0.1.0-SNAPSHOT",
Some(Files.ivy2Local))
Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-SNAPSHOT",
Some(Cache.ivy2Local))
}
}

View File

@ -1,4 +1,4 @@
com.github.alexarchambault:coursier_2.11:jar:0.1.0-SNAPSHOT
com.github.alexarchambault:coursier_2.11:jar:1.0.0-SNAPSHOT
org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4
org.scala-lang.modules:scala-xml_2.11:jar:1.0.4
org.scala-lang:scala-library:jar:2.11.7

View File

@ -4,7 +4,7 @@ package test
import utest._
import scala.async.Async.{ async, await }
import coursier.Fetch.default
import coursier.Platform.fetch
import coursier.test.compatibility._
object CentralTests extends TestSuite {
@ -44,7 +44,8 @@ object CentralTests extends TestSuite {
def resolutionCheck(
module: Module,
version: String,
extraRepo: Option[Repository] = None
extraRepo: Option[Repository] = None,
configuration: String = ""
) =
async {
val expected =
@ -54,7 +55,7 @@ object CentralTests extends TestSuite {
.split('\n')
.toSeq
val dep = Dependency(module, version)
val dep = Dependency(module, version, configuration = configuration)
val res = await(resolve(Set(dep), extraRepo = extraRepo))
val result = res
@ -138,6 +139,7 @@ object CentralTests extends TestSuite {
resolutionCheck(
Module("com.github.fommil", "java-logging"),
"1.2-SNAPSHOT",
configuration = "runtime",
extraRepo = Some(MavenRepository("https://oss.sonatype.org/content/repositories/public/"))
)
}

View File

@ -21,7 +21,7 @@ object PomParsingTests extends TestSuite {
</dependency>
"""
val expected = \/-(Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra")))
val expected = \/-("" -> Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra")))
val result = Pom.dependency(xmlParse(depNode).right.get)
@ -90,7 +90,7 @@ object PomParsingTests extends TestSuite {
None,
Profile.Activation(Nil),
Seq(
Dependency(Module("comp", "lib"), "0.2")),
"" -> Dependency(Module("comp", "lib"), "0.2")),
Nil,
Map.empty
))
@ -122,7 +122,7 @@ object PomParsingTests extends TestSuite {
Profile.Activation(Nil),
Nil,
Seq(
Dependency(Module("comp", "lib"), "0.2", scope = Scope.Test)),
"test" -> Dependency(Module("comp", "lib"), "0.2")),
Map.empty
))
@ -194,7 +194,7 @@ object PomParsingTests extends TestSuite {
val node = parsed.right.get
assert(node.label == "properties")
val children = node.child.collect{case elem if elem.isElement => elem}
val children = node.children.collect{case elem if elem.isElement => elem}
val props0 = children.toList.traverseU(Pom.property)
assert(props0.isRight)

View File

@ -2,6 +2,7 @@ package coursier
package test
import coursier.core.Repository
import coursier.maven.MavenRepository
import utest._
import scala.async.Async.{ async, await }
@ -16,7 +17,7 @@ object ResolutionTests extends TestSuite {
) =
Resolution(deps, filter = filter, forceVersions = forceVersions)
.process
.run(Fetch.default(repositories))
.run(Platform.fetch(repositories))
.runF
implicit class ProjectOps(val p: Project) extends AnyVal {
@ -27,69 +28,68 @@ object ResolutionTests extends TestSuite {
Project(Module("acme", "config"), "1.3.0"),
Project(Module("acme", "play"), "2.4.0", Seq(
Dependency(Module("acme", "play-json"), "2.4.0"))),
"" -> Dependency(Module("acme", "play-json"), "2.4.0"))),
Project(Module("acme", "play-json"), "2.4.0"),
Project(Module("acme", "play"), "2.4.1",
dependencies = Seq(
Dependency(Module("acme", "play-json"), "${playJsonVersion}"),
Dependency(Module("${project.groupId}", "${configName}"), "1.3.0")),
properties = Map(
"" -> Dependency(Module("acme", "play-json"), "${playJsonVersion}"),
"" -> Dependency(Module("${project.groupId}", "${configName}"), "1.3.0")),
properties = Seq(
"playJsonVersion" -> "2.4.0",
"configName" -> "config")),
Project(Module("acme", "play-extra-no-config"), "2.4.1",
Seq(
Dependency(Module("acme", "play"), "2.4.1",
"" -> Dependency(Module("acme", "play"), "2.4.1",
exclusions = Set(("acme", "config"))))),
Project(Module("acme", "play-extra-no-config-no"), "2.4.1",
Seq(
Dependency(Module("acme", "play"), "2.4.1",
"" -> Dependency(Module("acme", "play"), "2.4.1",
exclusions = Set(("*", "config"))))),
Project(Module("hudsucker", "mail"), "10.0",
Seq(
Dependency(Module("${project.groupId}", "test-util"), "${project.version}",
scope = Scope.Test))),
"test" -> Dependency(Module("${project.groupId}", "test-util"), "${project.version}"))),
Project(Module("hudsucker", "test-util"), "10.0"),
Project(Module("se.ikea", "parent"), "18.0",
dependencyManagement = Seq(
Dependency(Module("acme", "play"), "2.4.0",
"" -> Dependency(Module("acme", "play"), "2.4.0",
exclusions = Set(("acme", "play-json"))))),
Project(Module("se.ikea", "billy"), "18.0",
dependencies = Seq(
Dependency(Module("acme", "play"), "")),
"" -> Dependency(Module("acme", "play"), "")),
parent = Some(Module("se.ikea", "parent"), "18.0")),
Project(Module("org.gnome", "parent"), "7.0",
Seq(
Dependency(Module("org.gnu", "glib"), "13.4"))),
"" -> Dependency(Module("org.gnu", "glib"), "13.4"))),
Project(Module("org.gnome", "panel-legacy"), "7.0",
dependencies = Seq(
Dependency(Module("org.gnome", "desktop"), "${project.version}")),
"" -> Dependency(Module("org.gnome", "desktop"), "${project.version}")),
parent = Some(Module("org.gnome", "parent"), "7.0")),
Project(Module("gov.nsa", "secure-pgp"), "10.0",
Seq(
Dependency(Module("gov.nsa", "crypto"), "536.89"))),
"" -> Dependency(Module("gov.nsa", "crypto"), "536.89"))),
Project(Module("com.mailapp", "mail-client"), "2.1",
dependencies = Seq(
Dependency(Module("gov.nsa", "secure-pgp"), "10.0",
"" -> Dependency(Module("gov.nsa", "secure-pgp"), "10.0",
exclusions = Set(("*", "${crypto.name}")))),
properties = Map("crypto.name" -> "crypto", "dummy" -> "2")),
properties = Seq("crypto.name" -> "crypto", "dummy" -> "2")),
Project(Module("com.thoughtworks.paranamer", "paranamer-parent"), "2.6",
dependencies = Seq(
Dependency(Module("junit", "junit"), "")),
"" -> Dependency(Module("junit", "junit"), "")),
dependencyManagement = Seq(
Dependency(Module("junit", "junit"), "4.11", scope = Scope.Test))),
"test" -> Dependency(Module("junit", "junit"), "4.11"))),
Project(Module("com.thoughtworks.paranamer", "paranamer"), "2.6",
parent = Some(Module("com.thoughtworks.paranamer", "paranamer-parent"), "2.6")),
@ -97,75 +97,75 @@ object ResolutionTests extends TestSuite {
Project(Module("com.github.dummy", "libb"), "0.3.3",
profiles = Seq(
Profile("default", activeByDefault = Some(true), dependencies = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
"" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
Project(Module("com.github.dummy", "libb"), "0.4.2",
dependencies = Seq(
Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4")),
"" -> Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4")),
profiles = Seq(
Profile("default", activeByDefault = Some(true), dependencies = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"),
Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4", scope = Scope.Test))))),
"" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"),
"test" -> Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4"))))),
Project(Module("com.github.dummy", "libb"), "0.5.3",
properties = Map("special" -> "true"),
properties = Seq("special" -> "true"),
profiles = Seq(
Profile("default", activation = Profile.Activation(properties = Seq("special" -> None)), dependencies = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
"" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
Project(Module("com.github.dummy", "libb"), "0.5.4",
properties = Map("special" -> "true"),
properties = Seq("special" -> "true"),
profiles = Seq(
Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("true"))), dependencies = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
"" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
Project(Module("com.github.dummy", "libb"), "0.5.5",
properties = Map("special" -> "true"),
properties = Seq("special" -> "true"),
profiles = Seq(
Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
"" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
Project(Module("com.github.dummy", "libb-parent"), "0.5.6",
properties = Map("special" -> "true")),
properties = Seq("special" -> "true")),
Project(Module("com.github.dummy", "libb"), "0.5.6",
parent = Some(Module("com.github.dummy", "libb-parent"), "0.5.6"),
properties = Map("special" -> "true"),
properties = Seq("special" -> "true"),
profiles = Seq(
Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
"" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))),
Project(Module("an-org", "a-name"), "1.0"),
Project(Module("an-org", "a-name"), "1.2"),
Project(Module("an-org", "a-lib"), "1.0",
Seq(Dependency(Module("an-org", "a-name"), "1.0"))),
Seq("" -> Dependency(Module("an-org", "a-name"), "1.0"))),
Project(Module("an-org", "a-lib"), "1.1"),
Project(Module("an-org", "a-lib"), "1.2",
Seq(Dependency(Module("an-org", "a-name"), "1.2"))),
Seq("" -> Dependency(Module("an-org", "a-name"), "1.2"))),
Project(Module("an-org", "another-lib"), "1.0",
Seq(Dependency(Module("an-org", "a-name"), "1.0"))),
Seq("" -> Dependency(Module("an-org", "a-name"), "1.0"))),
// Must bring transitively an-org:a-name, as an optional dependency
Project(Module("an-org", "an-app"), "1.0",
Seq(
Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))),
Dependency(Module("an-org", "another-lib"), "1.0", optional = true))),
"" -> Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))),
"" -> Dependency(Module("an-org", "another-lib"), "1.0", optional = true))),
Project(Module("an-org", "an-app"), "1.1",
Seq(
Dependency(Module("an-org", "a-lib"), "1.1"))),
"" -> Dependency(Module("an-org", "a-lib"), "1.1"))),
Project(Module("an-org", "an-app"), "1.2",
Seq(
Dependency(Module("an-org", "a-lib"), "1.2")))
"" -> Dependency(Module("an-org", "a-lib"), "1.2")))
)
val projectsMap = projects.map(p => p.moduleVersion -> p).toMap
val projectsMap = projects.map(p => p.moduleVersion -> p.copy(configurations = MavenRepository.defaultConfigurations)).toMap
val testRepository = new TestRepository(projectsMap)
val repositories = Seq[Repository](
@ -243,14 +243,11 @@ object ResolutionTests extends TestSuite {
)
val res = await(resolve0(
Set(dep)
))
)).copy(filter = None, projectCache = Map.empty)
val expected = Resolution(
rootDependencies = Set(dep),
dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope),
projectCache = Map(
projectsMap(dep.moduleVersion).kv
) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv)
dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope)
)
assert(res == expected)
@ -267,14 +264,11 @@ object ResolutionTests extends TestSuite {
)
val res = await(resolve0(
Set(dep)
))
)).copy(filter = None, projectCache = Map.empty)
val expected = Resolution(
rootDependencies = Set(dep),
dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope),
projectCache = Map(
projectsMap(dep.moduleVersion).kv
) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv)
dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope)
)
assert(res == expected)
@ -291,14 +285,11 @@ object ResolutionTests extends TestSuite {
)
val res = await(resolve0(
Set(dep)
))
)).copy(filter = None, projectCache = Map.empty)
val expected = Resolution(
rootDependencies = Set(dep),
dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope),
projectCache = Map(
projectsMap(dep.moduleVersion).kv
) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv)
dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope)
)
assert(res == expected)
@ -308,16 +299,12 @@ object ResolutionTests extends TestSuite {
async {
val dep = Dependency(Module("hudsucker", "mail"), "10.0")
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
)).copy(filter = None)
Set(dep)
)).copy(filter = None, projectCache = Map.empty)
val expected = Resolution(
rootDependencies = Set(dep),
dependencies = Set(dep.withCompileScope),
projectCache = Map(
projectsMap(dep.moduleVersion).kv
)
dependencies = Set(dep.withCompileScope)
)
assert(res == expected)
@ -331,8 +318,7 @@ object ResolutionTests extends TestSuite {
exclusions = Set(("acme", "play-json")))
)
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
Set(dep)
)).copy(filter = None, projectCache = Map.empty)
val expected = Resolution(
@ -350,8 +336,7 @@ object ResolutionTests extends TestSuite {
Dependency(Module("org.gnu", "glib"), "13.4"),
Dependency(Module("org.gnome", "desktop"), "7.0"))
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
Set(dep)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -368,8 +353,7 @@ object ResolutionTests extends TestSuite {
val trDeps = Seq(
Dependency(Module("gov.nsa", "secure-pgp"), "10.0", exclusions = Set(("*", "crypto"))))
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
Set(dep)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -384,8 +368,7 @@ object ResolutionTests extends TestSuite {
async {
val dep = Dependency(Module("com.thoughtworks.paranamer", "paranamer"), "2.6")
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
Set(dep)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -402,8 +385,7 @@ object ResolutionTests extends TestSuite {
val trDeps = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
Set(dep)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -422,8 +404,7 @@ object ResolutionTests extends TestSuite {
val trDeps = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
Set(dep)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -444,8 +425,7 @@ object ResolutionTests extends TestSuite {
val trDeps = Seq(
Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
Set(dep)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -466,7 +446,7 @@ object ResolutionTests extends TestSuite {
Dependency(Module("an-org", "a-name"), "1.0", optional = true))
val res = await(resolve0(
Set(dep),
filter = Some(_.scope == Scope.Compile)
filter = Some(_ => true)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -489,7 +469,7 @@ object ResolutionTests extends TestSuite {
Dependency(Module("an-org", "a-name"), "1.0", optional = true))
val res = await(resolve0(
deps,
filter = Some(_.scope == Scope.Compile)
filter = Some(_ => true)
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -511,8 +491,7 @@ object ResolutionTests extends TestSuite {
val res = await(resolve0(
deps,
forceVersions = depOverrides,
filter = Some(_.scope == Scope.Compile)
forceVersions = depOverrides
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -536,8 +515,7 @@ object ResolutionTests extends TestSuite {
val res = await(resolve0(
deps,
forceVersions = depOverrides,
filter = Some(_.scope == Scope.Compile)
forceVersions = depOverrides
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -563,8 +541,7 @@ object ResolutionTests extends TestSuite {
val res = await(resolve0(
deps,
forceVersions = depOverrides,
filter = Some(_.scope == Scope.Compile)
forceVersions = depOverrides
)).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty)
val expected = Resolution(
@ -585,9 +562,9 @@ object ResolutionTests extends TestSuite {
'propertySubstitution{
val res =
core.Resolution.withProperties(
Seq(Dependency(Module("a-company", "a-name"), "${a.property}")),
Seq("" -> Dependency(Module("a-company", "a-name"), "${a.property}")),
Map("a.property" -> "a-version"))
val expected = Seq(Dependency(Module("a-company", "a-name"), "a-version"))
val expected = Seq("" -> Dependency(Module("a-company", "a-name"), "a-version"))
assert(res == expected)
}

View File

@ -8,12 +8,16 @@ import scalaz.Scalaz._
class TestRepository(projects: Map[(Module, String), Project]) extends Repository {
val source = new core.Artifact.Source {
def artifacts(dependency: Dependency, project: Project) = ???
def artifacts(
dependency: Dependency,
project: Project,
overrideClassifiers: Option[Seq[String]]
) = ???
}
def find[F[_]](
module: Module,
version: String,
fetch: Repository.Fetch[F]
fetch: Fetch.Content[F]
)(implicit
F: Monad[F]
) =

View File

@ -3,7 +3,7 @@ package coursier
package object test {
implicit class DependencyOps(val underlying: Dependency) extends AnyVal {
def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile)
def withCompileScope: Dependency = underlying.copy(configuration = "compile")
}
object Profile {
@ -13,25 +13,51 @@ package object test {
core.Activation(properties)
}
def apply(id: String,
activeByDefault: Option[Boolean] = None,
activation: Activation = Activation(),
dependencies: Seq[Dependency] = Nil,
dependencyManagement: Seq[Dependency] = Nil,
properties: Map[String, String] = Map.empty) =
core.Profile(id, activeByDefault, activation, dependencies, dependencyManagement, properties)
def apply(
id: String,
activeByDefault: Option[Boolean] = None,
activation: Activation = Activation(),
dependencies: Seq[(String, Dependency)] = Nil,
dependencyManagement: Seq[(String, Dependency)] = Nil,
properties: Map[String, String] = Map.empty
) =
core.Profile(
id,
activeByDefault,
activation,
dependencies,
dependencyManagement,
properties
)
}
object Project {
def apply(module: Module,
version: String,
dependencies: Seq[Dependency] = Seq.empty,
parent: Option[ModuleVersion] = None,
dependencyManagement: Seq[Dependency] = Seq.empty,
properties: Map[String, String] = Map.empty,
profiles: Seq[Profile] = Seq.empty,
versions: Option[core.Versions] = None,
snapshotVersioning: Option[core.SnapshotVersioning] = None): Project =
core.Project(module, version, dependencies, parent, dependencyManagement, properties, profiles, versions, snapshotVersioning)
def apply(
module: Module,
version: String,
dependencies: Seq[(String, Dependency)] = Seq.empty,
parent: Option[ModuleVersion] = None,
dependencyManagement: Seq[(String, Dependency)] = Seq.empty,
configurations: Map[String, Seq[String]] = Map.empty,
properties: Seq[(String, String)] = Seq.empty,
profiles: Seq[Profile] = Seq.empty,
versions: Option[core.Versions] = None,
snapshotVersioning: Option[core.SnapshotVersioning] = None,
publications: Seq[(String, core.Publication)] = Nil
): Project =
core.Project(
module,
version,
dependencies,
configurations,
parent,
dependencyManagement,
properties,
profiles,
versions,
snapshotVersioning,
publications,
Info.empty
)
}
}

View File

@ -1 +1 @@
version in ThisBuild := "0.1.0-SNAPSHOT"
version in ThisBuild := "1.0.0-SNAPSHOT"

View File

@ -11,14 +11,12 @@ import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue
import org.scalajs.jquery.jQuery
import scala.concurrent.Future
import scalaz.concurrent.Task
import scala.scalajs.js
import js.Dynamic.{ global => g }
case class ResolutionOptions(
followOptional: Boolean = false,
keepTest: Boolean = false
followOptional: Boolean = false
)
case class State(
@ -37,12 +35,12 @@ class Backend($: BackendScope[Unit, State]) {
def fetch(
repositories: Seq[core.Repository],
fetch: Repository.Fetch[Task]
): ResolutionProcess.Fetch[Task] = {
fetch: Fetch.Content[Task]
): Fetch.Metadata[Task] = {
modVers => Task.gatherUnordered(
modVers.map { case (module, version) =>
Repository.find(repositories, module, version, fetch)
Fetch.find(repositories, module, version, fetch)
.run
.map((module, version) -> _)
}
@ -66,7 +64,7 @@ class Backend($: BackendScope[Unit, State]) {
Seq(
dep.module.organization,
dep.module.name,
dep.scope.name
dep.configuration
).mkString(":")
for {
@ -176,8 +174,7 @@ class Backend($: BackendScope[Unit, State]) {
val res = coursier.Resolution(
s.modules.toSet,
filter = Some(dep =>
(s.options.followOptional || !dep.optional) &&
(s.options.keepTest || dep.scope != Scope.Test)
s.options.followOptional || !dep.optional
)
)
@ -338,14 +335,6 @@ class Backend($: BackendScope[Unit, State]) {
)
)
}
def toggleTest(e: ReactEventI) = {
$.modState(s =>
s.copy(
options = s.options
.copy(keepTest = !s.options.keepTest)
)
)
}
}
}
@ -380,7 +369,7 @@ object App {
<.td(dep.module.name),
<.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")),
<.td(Seq[Seq[TagMod]](
if (dep.scope == Scope.Compile) Seq() else Seq(infoLabel(dep.scope.name)),
if (dep.configuration == "compile") Seq() else Seq(infoLabel(dep.configuration)),
if (dep.attributes.`type`.isEmpty || dep.attributes.`type` == "jar") Seq() else Seq(infoLabel(dep.attributes.`type`)),
if (dep.attributes.classifier.isEmpty) Seq() else Seq(infoLabel(dep.attributes.classifier)),
Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq,
@ -417,7 +406,10 @@ object App {
}
val sortedDeps = res.minDependencies.toList
.sortBy(dep => coursier.core.Module.unapply(dep.module).get)
.sortBy { dep =>
val (org, name, _) = coursier.core.Module.unapply(dep.module).get
(org, name)
}
<.table(^.`class` := "table",
<.thead(
@ -683,15 +675,6 @@ object App {
"Follow optional dependencies"
)
)
),
<.div(^.`class` := "checkbox",
<.label(
<.input(^.`type` := "checkbox",
^.onChange ==> backend.options.toggleTest,
if (options.keepTest) Seq(^.checked := "checked") else Seq(),
"Keep test dependencies"
)
)
)
)
}