diff --git a/build.sbt b/build.sbt index c1a5a3f99..da18bde5f 100644 --- a/build.sbt +++ b/build.sbt @@ -51,19 +51,18 @@ lazy val baseCommonSettings = Seq( organization := "com.github.alexarchambault", resolvers ++= Seq( "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases", - Resolver.sonatypeRepo("releases"), - Resolver.sonatypeRepo("snapshots") + Resolver.sonatypeRepo("releases") ), scalacOptions += "-target:jvm-1.7", javacOptions ++= Seq( "-source", "1.7", "-target", "1.7" - ) + ), + javacOptions in Keys.doc := Seq() ) lazy val commonSettings = baseCommonSettings ++ Seq( scalaVersion := "2.11.7", - crossScalaVersions := Seq("2.10.6", "2.11.7"), libraryDependencies ++= { if (scalaVersion.value startsWith "2.10.") Seq(compilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full)) @@ -72,7 +71,6 @@ lazy val commonSettings = baseCommonSettings ++ Seq( } ) - lazy val core = crossProject .settings(commonSettings: _*) .settings(publishingSettings: _*) @@ -128,17 +126,18 @@ lazy val tests = crossProject scalaJSStage in Global := FastOptStage ) -lazy val testsJvm = tests.jvm.dependsOn(files % "test") +lazy val testsJvm = tests.jvm.dependsOn(cache % "test") lazy val testsJs = tests.js.dependsOn(`fetch-js` % "test") -lazy val files = project +lazy val cache = project .dependsOn(coreJvm) .settings(commonSettings) .settings(publishingSettings) .settings( - name := "coursier-files", + name := "coursier-cache", libraryDependencies ++= Seq( - "org.scalaz" %% "scalaz-concurrent" % "7.1.2" + "org.scalaz" %% "scalaz-concurrent" % "7.1.2", + "com.lihaoyi" %% "ammonite-terminal" % "0.5.0" ) ) @@ -156,20 +155,19 @@ lazy val bootstrap = project artifactName0(sv, m, artifact) }, crossPaths := false, - autoScalaLibrary := false, - javacOptions in doc := Seq() + autoScalaLibrary := false ) lazy val cli = project - .dependsOn(coreJvm, files) + .dependsOn(coreJvm, cache) .settings(commonSettings) - .settings(noPublishSettings) + .settings(publishingSettings) .settings(packAutoSettings) .settings( name := "coursier-cli", libraryDependencies ++= Seq( - "com.github.alexarchambault" %% "case-app" % "1.0.0-SNAPSHOT", - "com.lihaoyi" %% "ammonite-terminal" % "0.5.0", + // beware - available only in 2.11 + "com.github.alexarchambault" %% "case-app" % "1.0.0-M1", "ch.qos.logback" % "logback-classic" % "1.1.3" ), resourceGenerators in Compile += packageBin.in(bootstrap).in(Compile).map { jar => @@ -208,8 +206,27 @@ lazy val web = project ) ) +lazy val doc = project + .dependsOn(coreJvm, cache) + .settings(commonSettings) + .settings(noPublishSettings) + .settings(tutSettings) + .settings( + tutSourceDirectory := baseDirectory.value, + tutTargetDirectory := baseDirectory.value / ".." + ) + +// Don't try to compile that if you're not in 2.10 +lazy val plugin = project + .dependsOn(coreJvm, cache) + .settings(baseCommonSettings) + .settings( + name := "coursier-sbt-plugin", + sbtPlugin := true + ) + lazy val `coursier` = project.in(file(".")) - .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web) + .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, cache, bootstrap, cli, web, doc) .settings(commonSettings) .settings(noPublishSettings) .settings(releaseSettings) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala new file mode 100644 index 000000000..e66bda7e1 --- /dev/null +++ b/cache/src/main/scala/coursier/Cache.scala @@ -0,0 +1,555 @@ +package coursier + +import java.net.{HttpURLConnection, URL} +import java.nio.channels.{ OverlappingFileLockException, FileLock } +import java.nio.file.{ StandardCopyOption, Files => NioFiles } +import java.security.MessageDigest +import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService} + +import scala.annotation.tailrec +import scalaz._ +import scalaz.concurrent.{ Task, Strategy } + +import java.io.{ Serializable => _, _ } + +object Cache { + + private def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = { + def local(url: String) = + if (url.startsWith("file:///")) + url.stripPrefix("file://") + else if (url.startsWith("file:/")) + url.stripPrefix("file:") + else { + val localPathOpt = cache.collectFirst { + case (base, cacheDir) if url.startsWith(base) => + cacheDir + "/" + url.stripPrefix(base) + } + + localPathOpt.getOrElse { + // FIXME Means we were handed an artifact from repositories other than the known ones + println(cache.mkString("\n")) + println(url) + ??? + } + } + + if (artifact.extra.contains("local")) + artifact + else + artifact.copy(extra = artifact.extra + ("local" -> + artifact.copy( + url = local(artifact.url), + checksumUrls = artifact.checksumUrls + .mapValues(local) + .toVector + .toMap, + extra = Map.empty + ) + )) + } + + private def readFullyTo( + in: InputStream, + out: OutputStream, + logger: Option[Logger], + url: String, + alreadyDownloaded: Long + ): Unit = { + + val b = Array.fill[Byte](bufferSize)(0) + + @tailrec + def helper(count: Long): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + out.flush() + logger.foreach(_.downloadProgress(url, count + read)) + helper(count + read) + } + } + + helper(alreadyDownloaded) + } + + private def withLockFor[T](file: File)(f: => FileError \/ T): FileError \/ T = { + val lockFile = new File(file.getParentFile, s"${file.getName}.lock") + + lockFile.getParentFile.mkdirs() + var out = new FileOutputStream(lockFile) + + try { + var lock: FileLock = null + try { + lock = out.getChannel.tryLock() + if (lock == null) + -\/(FileError.Locked(file)) + else + try f + finally { + lock.release() + lock = null + out.close() + out = null + lockFile.delete() + } + } + catch { + case e: OverlappingFileLockException => + -\/(FileError.Locked(file)) + } + finally if (lock != null) lock.release() + } finally if (out != null) out.close() + } + + private def downloading[T]( + url: String, + file: File, + logger: Option[Logger] + )( + f: => FileError \/ T + ): FileError \/ T = + try { + val o = new Object + val prev = urlLocks.putIfAbsent(url, o) + if (prev == null) { + logger.foreach(_.downloadingArtifact(url, file)) + + val res = + try f + catch { case e: Exception => + logger.foreach(_.downloadedArtifact(url, success = false)) + throw e + } + finally { + urlLocks.remove(url) + } + + logger.foreach(_.downloadedArtifact(url, success = true)) + + res + } else + -\/(FileError.ConcurrentDownload(url)) + } + catch { case e: Exception => + -\/(FileError.DownloadError(s"Caught $e (${e.getMessage})")) + } + + private def temporaryFile(file: File): File = { + val dir = file.getParentFile + val name = file.getName + new File(dir, s"$name.part") + } + + private val partialContentResponseCode = 206 + + private def download( + artifact: Artifact, + cache: Seq[(String, File)], + checksums: Set[String], + cachePolicy: CachePolicy, + pool: ExecutorService, + logger: Option[Logger] = None + ): Task[Seq[((File, String), FileError \/ Unit)]] = { + + implicit val pool0 = pool + + val artifact0 = withLocal(artifact, cache) + .extra + .getOrElse("local", artifact) + + val pairs = + Seq(artifact0.url -> artifact.url) ++ { + checksums + .intersect(artifact0.checksumUrls.keySet) + .intersect(artifact.checksumUrls.keySet) + .toSeq + .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) + } + + def urlConn(url: String) = { + val conn = new URL(url).openConnection() // FIXME Should this be closed? + // Dummy user-agent instead of the default "Java/...", + // so that we are not returned incomplete/erroneous metadata + // (Maven 2 compatibility? - happens for snapshot versioning metadata, + // this is SO FSCKING CRAZY) + conn.setRequestProperty("User-Agent", "") + conn + } + + + def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] = + EitherT { + Task { + \/- { + val lastModified = file.lastModified() + if (lastModified > 0L) + Some(lastModified) + else + None + } : FileError \/ Option[Long] + } + } + + def urlLastModified(url: String): EitherT[Task, FileError, Option[Long]] = + EitherT { + Task { + urlConn(url) match { + case c: HttpURLConnection => + c.setRequestMethod("HEAD") + val remoteLastModified = c.getLastModified + + \/- { + if (remoteLastModified > 0L) + Some(remoteLastModified) + else + None + } + + case other => + -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)")) + } + } + } + + def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] = + for { + fileLastModOpt <- fileLastModified(file) + urlLastModOpt <- urlLastModified(url) + } yield { + val fromDatesOpt = for { + fileLastMod <- fileLastModOpt + urlLastMod <- urlLastModOpt + } yield fileLastMod < urlLastMod + + fromDatesOpt.getOrElse(true) + } + + def remote(file: File, url: String): EitherT[Task, FileError, Unit] = + EitherT { + Task { + withLockFor(file) { + downloading(url, file, logger) { + val tmp = temporaryFile(file) + + val alreadyDownloaded = tmp.length() + + val conn0 = urlConn(url) + + val (partialDownload, conn) = conn0 match { + case conn0: HttpURLConnection if alreadyDownloaded > 0L => + conn0.setRequestProperty("Range", s"bytes=$alreadyDownloaded-") + + if (conn0.getResponseCode == partialContentResponseCode) { + val ackRange = Option(conn0.getHeaderField("Content-Range")).getOrElse("") + + if (ackRange.startsWith(s"bytes $alreadyDownloaded-")) + (true, conn0) + else + // unrecognized Content-Range header -> start a new connection with no resume + (false, urlConn(url)) + } else + (false, conn0) + + case _ => (false, conn0) + } + + for (len0 <- Option(conn.getContentLengthLong) if len0 >= 0L) { + val len = len0 + (if (partialDownload) alreadyDownloaded else 0L) + logger.foreach(_.downloadLength(url, len)) + } + + val in = new BufferedInputStream(conn.getInputStream, bufferSize) + + val result = + try { + tmp.getParentFile.mkdirs() + val out = new FileOutputStream(tmp, partialDownload) + try \/-(readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)) + finally out.close() + } finally in.close() + + file.getParentFile.mkdirs() + NioFiles.move(tmp.toPath, file.toPath, StandardCopyOption.ATOMIC_MOVE) + + for (lastModified <- Option(conn.getLastModified) if lastModified > 0L) + file.setLastModified(lastModified) + + result + } + } + } + } + + def checkFileExists(file: File, url: String): EitherT[Task, FileError, Unit] = + EitherT { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(url, file)) + \/-(()) + } else + -\/(FileError.NotFound(file.toString)) + } + } + + val tasks = + for ((f, url) <- pairs) yield { + val file = new File(f) + + val res = + if (url.startsWith("file:/")) { + def filtered(s: String) = + s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/") + assert( + filtered(url) == filtered(file.toURI.toString), + s"URL: ${filtered(url)}, file: ${filtered(file.toURI.toString)}" + ) + checkFileExists(file, url) + } else + cachePolicy match { + case CachePolicy.LocalOnly => + checkFileExists(file, url) + case CachePolicy.UpdateChanging | CachePolicy.Update => + shouldDownload(file, url).flatMap { + case true => + remote(file, url) + case false => + EitherT(Task.now(\/-(()) : FileError \/ Unit)) + } + case CachePolicy.FetchMissing => + checkFileExists(file, url) orElse remote(file, url) + case CachePolicy.ForceDownload => + remote(file, url) + } + + res.run.map((file, url) -> _) + } + + Nondeterminism[Task].gather(tasks) + } + + def validateChecksum( + artifact: Artifact, + sumType: String, + cache: Seq[(String, File)], + pool: ExecutorService + ): EitherT[Task, FileError, Unit] = { + + implicit val pool0 = pool + + val artifact0 = withLocal(artifact, cache) + .extra + .getOrElse("local", artifact) + + EitherT { + artifact0.checksumUrls.get(sumType) match { + case Some(sumFile) => + Task { + val sum = scala.io.Source.fromFile(sumFile) + .getLines() + .toStream + .headOption + .mkString + .takeWhile(!_.isSpaceChar) + + val f = new File(artifact0.url) + val md = MessageDigest.getInstance(sumType) + val is = new FileInputStream(f) + val res = try { + var lock: FileLock = null + try { + lock = is.getChannel.tryLock(0L, Long.MaxValue, true) + if (lock == null) + -\/(FileError.Locked(f)) + else { + withContent(is, md.update(_, 0, _)) + \/-(()) + } + } + catch { + case e: OverlappingFileLockException => + -\/(FileError.Locked(f)) + } + finally if (lock != null) lock.release() + } finally is.close() + + res.flatMap { _ => + val digest = md.digest() + val calculatedSum = f"${BigInt(1, digest)}%040x" + + if (sum == calculatedSum) + \/-(()) + else + -\/(FileError.WrongChecksum(sumType, calculatedSum, sum, artifact0.url, sumFile)) + } + } + + case None => + Task.now(-\/(FileError.ChecksumNotFound(sumType, artifact0.url))) + } + } + } + + def file( + artifact: Artifact, + cache: Seq[(String, File)], + cachePolicy: CachePolicy, + checksums: Seq[Option[String]] = Seq(Some("SHA-1")), + logger: Option[Logger] = None, + pool: ExecutorService = defaultPool + ): EitherT[Task, FileError, File] = { + + implicit val pool0 = pool + + val checksums0 = if (checksums.isEmpty) Seq(None) else checksums + + val res = EitherT { + download( + artifact, + cache, + checksums = checksums0.collect { case Some(c) => c }.toSet, + cachePolicy, + pool, + logger = logger + ).map { results => + val checksum = checksums0.find { + case None => true + case Some(c) => + artifact.checksumUrls.get(c).exists { cUrl => + results.exists { case ((_, u), b) => + u == cUrl && b.isRight + } + } + } + + val ((f, _), res) = results.head + res.flatMap { _ => + checksum match { + case None => + // FIXME All the checksums should be in the error, possibly with their URLs + // from artifact.checksumUrls + -\/(FileError.ChecksumNotFound(checksums0.last.get, "")) + case Some(c) => \/-((f, c)) + } + } + } + } + + res.flatMap { + case (f, None) => EitherT(Task.now[FileError \/ File](\/-(f))) + case (f, Some(c)) => + validateChecksum(artifact, c, cache, pool).map(_ => f) + } + } + + def fetch( + cache: Seq[(String, File)], + cachePolicy: CachePolicy, + checksums: Seq[Option[String]] = Seq(Some("SHA-1")), + logger: Option[Logger] = None, + pool: ExecutorService = defaultPool + ): Fetch.Content[Task] = { + artifact => + file( + artifact, + cache, + cachePolicy, + checksums = checksums, + logger = logger, + pool = pool + ).leftMap(_.message).map { f => + // FIXME Catch error here? + scala.io.Source.fromFile(f)("UTF-8").mkString + } + } + + lazy val ivy2Local = MavenRepository( + new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, + ivyLike = true + ) + + val defaultConcurrentDownloadCount = 6 + + lazy val defaultPool = + Executors.newFixedThreadPool(defaultConcurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) + + + private val urlLocks = new ConcurrentHashMap[String, Object] + + trait Logger { + def foundLocally(url: String, f: File): Unit = {} + def downloadingArtifact(url: String, file: File): Unit = {} + def downloadLength(url: String, length: Long): Unit = {} + def downloadProgress(url: String, downloaded: Long): Unit = {} + def downloadedArtifact(url: String, success: Boolean): Unit = {} + } + + var bufferSize = 1024*1024 + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap(_.getMessage) + } + + def withContent(is: InputStream, f: (Array[Byte], Int) => Unit): Unit = { + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + f(data, nRead) + nRead = is.read(data, 0, data.length) + } + } + +} + +sealed trait FileError extends Product with Serializable { + def message: String +} + +object FileError { + + case class DownloadError(message0: String) extends FileError { + def message = s"Download error: $message0" + } + case class NotFound(file: String) extends FileError { + def message = s"$file: not found" + } + case class ChecksumNotFound(sumType: String, file: String) extends FileError { + def message = s"$file: $sumType checksum not found" + } + case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError { + def message = s"$file: $sumType checksum validation failed" + } + + sealed trait Recoverable extends FileError + case class Locked(file: File) extends Recoverable { + def message = s"$file: locked" + } + case class ConcurrentDownload(url: String) extends Recoverable { + def message = s"$url: concurrent download" + } + +} diff --git a/cache/src/main/scala/coursier/CachePolicy.scala b/cache/src/main/scala/coursier/CachePolicy.scala new file mode 100644 index 000000000..0bf51e6e3 --- /dev/null +++ b/cache/src/main/scala/coursier/CachePolicy.scala @@ -0,0 +1,11 @@ +package coursier + +sealed trait CachePolicy extends Product with Serializable + +object CachePolicy { + case object LocalOnly extends CachePolicy + case object UpdateChanging extends CachePolicy + case object Update extends CachePolicy + case object FetchMissing extends CachePolicy + case object ForceDownload extends CachePolicy +} diff --git a/files/src/main/scala/coursier/Platform.scala b/cache/src/main/scala/coursier/Platform.scala similarity index 87% rename from files/src/main/scala/coursier/Platform.scala rename to cache/src/main/scala/coursier/Platform.scala index c376fefe3..375372f6b 100644 --- a/files/src/main/scala/coursier/Platform.scala +++ b/cache/src/main/scala/coursier/Platform.scala @@ -39,7 +39,7 @@ object Platform { } } - val artifact: Repository.Fetch[Task] = { artifact => + val artifact: Fetch.Content[Task] = { artifact => EitherT { val url = new URL(artifact.url) @@ -53,4 +53,9 @@ object Platform { } } + implicit def fetch( + repositories: Seq[core.Repository] + ): Fetch.Metadata[Task] = + Fetch(repositories, Platform.artifact) + } diff --git a/cache/src/main/scala/coursier/TermDisplay.scala b/cache/src/main/scala/coursier/TermDisplay.scala new file mode 100644 index 000000000..4447731db --- /dev/null +++ b/cache/src/main/scala/coursier/TermDisplay.scala @@ -0,0 +1,234 @@ +package coursier + +import java.io.{File, Writer} +import java.util.concurrent._ + +import ammonite.terminal.{ TTY, Ansi } + +import scala.annotation.tailrec +import scala.collection.mutable.ArrayBuffer + +class TermDisplay( + out: Writer, + var fallbackMode: Boolean = false +) extends Cache.Logger { + + private val ansi = new Ansi(out) + private var width = 80 + private val refreshInterval = 1000 / 60 + private val fallbackRefreshInterval = 1000 + + private val lock = new AnyRef + private var currentHeight = 0 + private val t = new Thread("TermDisplay") { + override def run() = lock.synchronized { + + val baseExtraWidth = width / 5 + + def reflowed(url: String, info: Info) = { + val pctOpt = info.pct.map(100.0 * _) + val extra = + if (info.length.isEmpty && info.downloaded == 0L) + "" + else + s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${info.downloaded}${info.length.map(" / " + _).mkString})" + + val total = url.length + 1 + extra.length + val (url0, extra0) = + if (total >= width) { // or > ? If equal, does it go down 2 lines? + val overflow = total - width + 1 + + val extra0 = + if (extra.length > baseExtraWidth) + extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…" + else + extra + + val total0 = url.length + 1 + extra0.length + val overflow0 = total0 - width + 1 + + val url0 = + if (total0 >= width) + url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…" + else + url + + (url0, extra0) + } else + (url, extra) + + (url0, extra0) + } + + + @tailrec def helper(lineCount: Int): Unit = { + currentHeight = lineCount + + Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { + case None => helper(lineCount) + case Some(Left(())) => // poison pill + case Some(Right(())) => + // update display + + val downloads0 = downloads.synchronized { + downloads + .toVector + .map { url => url -> infos.get(url) } + .sortBy { case (_, info) => - info.pct.sum } + } + + for ((url, info) <- downloads0) { + assert(info != null, s"Incoherent state ($url)") + + val (url0, extra0) = reflowed(url, info) + + ansi.clearLine(2) + out.write(s"$url0 $extra0\n") + } + + if (downloads0.length < lineCount) { + for (_ <- downloads0.length until lineCount) { + ansi.clearLine(2) + ansi.down(1) + } + + for (_ <- downloads0.length until lineCount) + ansi.up(1) + } + + for (_ <- downloads0.indices) + ansi.up(1) + + out.flush() + Thread.sleep(refreshInterval) + helper(downloads0.length) + } + } + + + @tailrec def fallbackHelper(previous: Set[String]): Unit = + Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { + case None => fallbackHelper(previous) + case Some(Left(())) => // poison pill + case Some(Right(())) => + val downloads0 = downloads.synchronized { + downloads + .toVector + .map { url => url -> infos.get(url) } + .sortBy { case (_, info) => - info.pct.sum } + } + + var displayedSomething = false + for ((url, info) <- downloads0 if previous(url)) { + assert(info != null, s"Incoherent state ($url)") + + val (url0, extra0) = reflowed(url, info) + + displayedSomething = true + out.write(s"$url0 $extra0\n") + } + + if (displayedSomething) + out.write("\n") + + out.flush() + Thread.sleep(fallbackRefreshInterval) + fallbackHelper(previous ++ downloads0.map { case (url, _) => url }) + } + + if (fallbackMode) + fallbackHelper(Set.empty) + else + helper(0) + } + } + + t.setDaemon(true) + + def init(): Unit = { + try { + width = TTY.consoleDim("cols") + ansi.clearLine(2) + } catch { case _: Exception => + fallbackMode = true + } + + t.start() + } + + def stop(): Unit = { + for (_ <- 0 until currentHeight) { + ansi.clearLine(2) + ansi.down(1) + } + for (_ <- 0 until currentHeight) { + ansi.up(1) + } + q.put(Left(())) + lock.synchronized(()) + } + + private case class Info(downloaded: Long, length: Option[Long]) { + def pct: Option[Double] = length.map(downloaded.toDouble / _) + } + + private val downloads = new ArrayBuffer[String] + private val infos = new ConcurrentHashMap[String, Info] + + private val q = new LinkedBlockingDeque[Either[Unit, Unit]] + def update(): Unit = { + if (q.size() == 0) + q.put(Right(())) + } + + override def downloadingArtifact(url: String, file: File): Unit = { + assert(!infos.containsKey(url)) + val prev = infos.putIfAbsent(url, Info(0L, None)) + assert(prev == null) + + if (fallbackMode) { + // FIXME What about concurrent accesses to out from the thread above? + out.write(s"Downloading $url\n") + out.flush() + } + + downloads.synchronized { + downloads.append(url) + } + + update() + } + override def downloadLength(url: String, length: Long): Unit = { + val info = infos.get(url) + assert(info != null) + val newInfo = info.copy(length = Some(length)) + infos.put(url, newInfo) + + update() + } + override def downloadProgress(url: String, downloaded: Long): Unit = { + val info = infos.get(url) + assert(info != null) + val newInfo = info.copy(downloaded = downloaded) + infos.put(url, newInfo) + + update() + } + override def downloadedArtifact(url: String, success: Boolean): Unit = { + downloads.synchronized { + downloads -= url + } + + if (fallbackMode) { + // FIXME What about concurrent accesses to out from the thread above? + out.write(s"Downloaded $url\n") + out.flush() + } + + val info = infos.remove(url) + assert(info != null) + + update() + } + +} diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 9dd707773..4bc3d55d5 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -12,12 +12,10 @@ import coursier.util.ClasspathFilter case class CommonOptions( @HelpMessage("Keep optional dependencies (Maven)") keepOptional: Boolean, - @HelpMessage("Off-line mode: only use cache and local repositories") - @ExtraName("c") - offline: Boolean, - @HelpMessage("Force download: for remote repositories only: re-download items, that is, don't use cache directly") - @ExtraName("f") - force: Boolean, + @HelpMessage("Download mode (default: missing, that is fetch things missing from cache)") + @ValueDescription("offline|update-changing|update|missing|force") + @ExtraName("m") + mode: String = "missing", @HelpMessage("Quiet output") @ExtraName("q") quiet: Boolean, @@ -30,6 +28,10 @@ case class CommonOptions( @HelpMessage("Repositories - for multiple repositories, separate with comma and/or repeat this option (e.g. -r central,ivy2local -r sonatype-snapshots, or equivalently -r central,ivy2local,sonatype-snapshots)") @ExtraName("r") repository: List[String], + @HelpMessage("Do not add default repositories (~/.ivy2/local, and Central)") + noDefault: Boolean = false, + @HelpMessage("Modify names in Maven repository paths for SBT plugins") + sbtPluginHack: Boolean = false, @HelpMessage("Force module version") @ValueDescription("organization:name:forcedVersion") @ExtraName("V") @@ -66,19 +68,28 @@ case class Fetch( @HelpMessage("Fetch javadoc artifacts") @ExtraName("D") javadoc: Boolean, + @HelpMessage("Print java -cp compatible output") + @ExtraName("p") + classpath: Boolean, @Recurse common: CommonOptions ) extends CoursierCommand { val helper = new Helper(common, remainingArgs) - val files0 = helper.fetch(main = true, sources = false, javadoc = false) + val files0 = helper.fetch(sources = sources, javadoc = javadoc) - println( - files0 - .map(_.toString) - .mkString("\n") - ) + val out = + if (classpath) + files0 + .map(_.toString) + .mkString(File.pathSeparator) + else + files0 + .map(_.toString) + .mkString("\n") + + println(out) } @@ -86,6 +97,9 @@ case class Launch( @ExtraName("M") @ExtraName("main") mainClass: String, + @ExtraName("c") + @HelpMessage("Assume coursier is a dependency of the launched app, and share the coursier dependency of the launcher with it - allows the launched app to get the resolution that launched it via ResolutionClassLoader") + addCoursier: Boolean, @Recurse common: CommonOptions ) extends CoursierCommand { @@ -99,15 +113,37 @@ case class Launch( } } - val helper = new Helper(common, rawDependencies) + val extraForceVersions = + if (addCoursier) + ??? + else + Seq.empty[String] - val files0 = helper.fetch(main = true, sources = false, javadoc = false) + val dontFilterOut = + if (addCoursier) { + val url = classOf[coursier.core.Resolution].getProtectionDomain.getCodeSource.getLocation + + if (url.getProtocol == "file") + Seq(new File(url.getPath)) + else { + Console.err.println(s"Cannot get the location of the JAR of coursier ($url not a file URL)") + sys.exit(255) + } + } else + Seq.empty[File] + + val helper = new Helper( + common.copy(forceVersion = common.forceVersion ++ extraForceVersions), + rawDependencies + ) + + val files0 = helper.fetch(sources = false, javadoc = false) val cl = new URLClassLoader( files0.map(_.toURI.toURL).toArray, new ClasspathFilter( Thread.currentThread().getContextClassLoader, - Coursier.baseCp.map(new File(_)).toSet, + Coursier.baseCp.map(new File(_)).toSet -- dontFilterOut, exclude = true ) ) @@ -169,108 +205,6 @@ case class Launch( method.invoke(null, extraArgs.toArray) } -case class Classpath( - @Recurse - common: CommonOptions -) extends CoursierCommand { - - val helper = new Helper(common, remainingArgs) - - val files0 = helper.fetch(main = true, sources = false, javadoc = false) - - Console.out.println( - files0 - .map(_.toString) - .mkString(File.pathSeparator) - ) - -} - -// TODO: allow removing a repository (with confirmations, etc.) -case class Repository( - @ValueDescription("id:baseUrl") - @ExtraName("a") - add: List[String], - @ExtraName("L") - list: Boolean, - @ExtraName("l") - defaultList: Boolean, - ivyLike: Boolean, - @Recurse - cacheOptions: CacheOptions -) extends CoursierCommand { - - if (add.exists(!_.contains(":"))) { - CaseApp.printUsage[Repository](err = true) - sys.exit(255) - } - - val add0 = add - .map{ s => - val Seq(id, baseUrl) = s.split(":", 2).toSeq - id -> baseUrl - } - - if ( - add0.exists(_._1.contains("/")) || - add0.exists(_._1.startsWith(".")) || - add0.exists(_._1.isEmpty) - ) { - CaseApp.printUsage[Repository](err = true) - sys.exit(255) - } - - - val cache = Cache(new File(cacheOptions.cache)) - - if (cache.cache.exists() && !cache.cache.isDirectory) { - Console.err.println(s"Error: ${cache.cache} not a directory") - sys.exit(1) - } - - if (!cache.cache.exists()) - cache.init(verbose = true) - - val current = cache.list().map(_._1).toSet - - val alreadyAdded = add0 - .map(_._1) - .filter(current) - - if (alreadyAdded.nonEmpty) { - Console.err.println(s"Error: already added: ${alreadyAdded.mkString(", ")}") - sys.exit(1) - } - - for ((id, baseUrl0) <- add0) { - val baseUrl = - if (baseUrl0.endsWith("/")) - baseUrl0 - else - baseUrl0 + "/" - - cache.add(id, baseUrl, ivyLike = ivyLike) - } - - if (defaultList) { - val map = cache.repositoryMap() - - for (id <- cache.default(withNotFound = true)) - map.get(id) match { - case Some(repo) => - println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) - case None => - println(s"$id (not found)") - } - } - - if (list) - for ((id, repo, _) <- cache.list().sortBy(_._1)) { - println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) - } - -} - case class Bootstrap( @ExtraName("M") @ExtraName("main") @@ -325,7 +259,7 @@ case class Bootstrap( val bootstrapJar = Option(Thread.currentThread().getContextClassLoader.getResourceAsStream("bootstrap.jar")) match { - case Some(is) => Files.readFullySync(is) + case Some(is) => Cache.readFullySync(is) case None => Console.err.println(s"Error: bootstrap JAR not found") sys.exit(1) diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index db82754d4..5bc66ff75 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -2,28 +2,14 @@ package coursier package cli import java.io.{ OutputStreamWriter, File } -import java.util.UUID +import java.util.concurrent.Executors + +import coursier.ivy.IvyRepository import scalaz.{ \/-, -\/ } -import scalaz.concurrent.Task +import scalaz.concurrent.{ Task, Strategy } object Helper { - def validate(common: CommonOptions) = { - import common._ - - if (force && offline) { - Console.err.println("Error: --offline (-c) and --force (-f) options can't be specified at the same time.") - sys.exit(255) - } - - if (parallel <= 0) { - Console.err.println(s"Error: invalid --parallel (-n) value: $parallel") - sys.exit(255) - } - - ??? - } - def fileRepr(f: File) = f.toString def errPrintln(s: String) = Console.err.println(s) @@ -54,77 +40,91 @@ class Helper( import common._ import Helper.errPrintln - implicit val cachePolicy = - if (offline) - CachePolicy.LocalOnly - else if (force) - CachePolicy.ForceDownload - else - CachePolicy.Default - - val cache = Cache(new File(cacheOptions.cache)) - cache.init(verbose = verbose0 >= 0) - - val repositoryIds = { - val repositoryIds0 = repository - .flatMap(_.split(',')) - .map(_.trim) - .filter(_.nonEmpty) - - if (repositoryIds0.isEmpty) - cache.default() - else - repositoryIds0 + val cachePolicies = mode match { + case "offline" => + Seq(CachePolicy.LocalOnly) + case "update-changing" => + Seq(CachePolicy.UpdateChanging) + case "update" => + Seq(CachePolicy.Update) + case "missing" => + Seq(CachePolicy.FetchMissing) + case "force" => + Seq(CachePolicy.ForceDownload) + case "default" => + Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing) + case other => + errPrintln(s"Unrecognized mode: $other") + sys.exit(255) } - val repoMap = cache.map() - val repoByBase = repoMap.map { case (_, v @ (m, _)) => - m.root -> v - } + val caches = + Seq( + "http://" -> new File(new File(cacheOptions.cache), "http"), + "https://" -> new File(new File(cacheOptions.cache), "https") + ) - val repositoryIdsOpt0 = repositoryIds.map { id => - repoMap.get(id) match { - case Some(v) => Right(v) - case None => - if (id.contains("://")) { - val root0 = if (id.endsWith("/")) id else id + "/" - Right( - repoByBase.getOrElse(root0, { - val id0 = UUID.randomUUID().toString - if (verbose0 >= 1) - Console.err.println(s"Addding repository $id0 ($root0)") + val pool = Executors.newFixedThreadPool(parallel, Strategy.DefaultDaemonThreadFactory) - // FIXME This could be done more cleanly - cache.add(id0, root0, ivyLike = false) - cache.map().getOrElse(id0, - sys.error(s"Adding repository $id0 ($root0)") - ) - }) - ) - } else - Left(id) + val central = MavenRepository("https://repo1.maven.org/maven2/") + val ivy2Local = MavenRepository( + new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, + ivyLike = true + ) + val defaultRepositories = Seq( + ivy2Local, + central + ) + + val repositories0 = common.repository.map { repo => + val repo0 = repo.toLowerCase + if (repo0 == "central") + Right(central) + else if (repo0 == "ivy2local") + Right(ivy2Local) + else if (repo0.startsWith("sonatype:")) + Right( + MavenRepository(s"https://oss.sonatype.org/content/repositories/${repo.drop("sonatype:".length)}") + ) + else { + val (url, r) = + if (repo.startsWith("ivy:")) { + val url = repo.drop("ivy:".length) + (url, IvyRepository(url)) + } else if (repo.startsWith("ivy-like:")) { + val url = repo.drop("ivy-like:".length) + (url, MavenRepository(url, ivyLike = true)) + } else { + (repo, MavenRepository(repo)) + } + + if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/")) + Right(r) + else + Left(repo -> s"Unrecognized protocol or repository: $url") } } - val notFoundRepositoryIds = repositoryIdsOpt0.collect { - case Left(id) => id - } - - if (notFoundRepositoryIds.nonEmpty) { - errPrintln( - (if (notFoundRepositoryIds.lengthCompare(1) == 0) "Repository" else "Repositories") + - " not found: " + - notFoundRepositoryIds.mkString(", ") - ) - + val unrecognizedRepos = repositories0.collect { case Left(e) => e } + if (unrecognizedRepos.nonEmpty) { + errPrintln(s"${unrecognizedRepos.length} error(s) parsing repositories:") + for ((repo, err) <- unrecognizedRepos) + errPrintln(s"$repo: $err") sys.exit(255) } - val files = cache.files().copy(concurrentDownloadCount = parallel) + val repositories1 = + (if (common.noDefault) Nil else defaultRepositories) ++ + repositories0.collect { case Right(r) => r } - val (repositories, fileCaches) = repositoryIdsOpt0 - .collect { case Right(v) => v } - .unzip + val repositories = + if (common.sbtPluginHack) + repositories1.map { + case m: MavenRepository => m.copy(sbtAttrStub = true) + case other => other + } + else + repositories1 val (rawDependencies, extraArgs) = { val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) @@ -166,12 +166,25 @@ class Helper( } val moduleVersions = splitDependencies.map{ - case Seq(org, name, version) => - (Module(org, name), version) + case Seq(org, namePart, version) => + val p = namePart.split(';') + val name = p.head + val splitAttributes = p.tail.map(_.split("=", 2).toSeq).toSeq + val malformedAttributes = splitAttributes.filter(_.length != 2) + if (malformedAttributes.nonEmpty) { + // FIXME Get these for all dependencies at once + Console.err.println(s"Malformed attributes in ${splitDependencies.mkString(":")}") + // :( + sys.exit(255) + } + val attributes = splitAttributes.collect { + case Seq(k, v) => k -> v + } + (Module(org, name, attributes.toMap), version) } val deps = moduleVersions.map{case (mod, ver) => - Dependency(mod, ver, scope = Scope.Runtime) + Dependency(mod, ver, configuration = "runtime") } val forceVersions = { @@ -200,11 +213,14 @@ class Helper( Some(new TermDisplay(new OutputStreamWriter(System.err))) else None - logger.foreach(_.init()) + + val fetchs = cachePolicies.map(p => + Cache.fetch(caches, p, logger = logger, pool = pool) + ) val fetchQuiet = coursier.Fetch( repositories, - files.fetch(logger = logger)(cachePolicy = CachePolicy.LocalOnly), // local files get the priority - files.fetch(logger = logger) + fetchs.head, + fetchs.tail: _* ) val fetch0 = if (verbose0 <= 0) fetchQuiet @@ -229,6 +245,7 @@ class Helper( } } + logger.foreach(_.init()) val res = startRes .process @@ -296,35 +313,47 @@ class Helper( } } - def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = { - if (verbose0 >= 0) - errPrintln("Fetching artifacts") - val artifacts0 = res.artifacts - val main0 = main || (!sources && !javadoc) - val artifacts = artifacts0.flatMap{ artifact => - var l = List.empty[Artifact] - if (sources) - l = artifact.extra.get("sources").toList ::: l - if (javadoc) - l = artifact.extra.get("javadoc").toList ::: l - if (main0) - l = artifact :: l + def fetch(sources: Boolean, javadoc: Boolean): Seq[File] = { + if (verbose0 >= 0) { + val msg = cachePolicies match { + case Seq(CachePolicy.LocalOnly) => + "Checking artifacts" + case _ => + "Fetching artifacts" + } - l + errPrintln(msg) } + val artifacts = + if (sources || javadoc) { + var classifiers = Seq.empty[String] + if (sources) + classifiers = classifiers :+ "sources" + if (javadoc) + classifiers = classifiers :+ "javadoc" + + res.classifiersArtifacts(classifiers) + } else + res.artifacts val logger = if (verbose0 >= 0) Some(new TermDisplay(new OutputStreamWriter(System.err))) else None + + if (verbose0 >= 1 && artifacts.nonEmpty) + println(s"Found ${artifacts.length} artifacts") + + val tasks = artifacts.map(artifact => + (Cache.file(artifact, caches, cachePolicies.head, logger = logger, pool = pool) /: cachePolicies.tail)( + _ orElse Cache.file(artifact, caches, _, logger = logger, pool = pool) + ).run.map(artifact.->) + ) + logger.foreach(_.init()) - val tasks = artifacts.map(artifact => files.file(artifact, logger = logger).run.map(artifact.->)) - def printTask = Task { - if (verbose0 >= 1 && artifacts.nonEmpty) - println(s"Found ${artifacts.length} artifacts") - } - val task = printTask.flatMap(_ => Task.gatherUnordered(tasks)) + + val task = Task.gatherUnordered(tasks) val results = task.run val errors = results.collect{case (artifact, -\/(err)) => artifact -> err } diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cli/src/main/scala/coursier/cli/TermDisplay.scala deleted file mode 100644 index 62f98adc3..000000000 --- a/cli/src/main/scala/coursier/cli/TermDisplay.scala +++ /dev/null @@ -1,150 +0,0 @@ -package coursier.cli - -import java.io.Writer -import java.util.concurrent._ - -import ammonite.terminal.{ TTY, Ansi } - -import coursier.Files.Logger - -import scala.annotation.tailrec -import scala.collection.mutable.ArrayBuffer - -class TermDisplay(out: Writer) extends Logger { - - private val ansi = new Ansi(out) - private var width = 80 - private val refreshInterval = 1000 / 60 - private val lock = new AnyRef - private val t = new Thread("TermDisplay") { - override def run() = lock.synchronized { - val baseExtraWidth = width / 5 - @tailrec def helper(lineCount: Int): Unit = - Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { - case None => helper(lineCount) - case Some(Left(())) => // poison pill - case Some(Right(())) => - // update display - - for (_ <- 0 until lineCount) { - ansi.up(1) - ansi.clearLine(2) - } - - val downloads0 = downloads.synchronized { - downloads - .toVector - .map { url => url -> infos.get(url) } - .sortBy { case (_, info) => - info.pct.sum } - } - - for ((url, info) <- downloads0) { - assert(info != null, s"Incoherent state ($url)") - val pctOpt = info.pct.map(100.0 * _) - val extra = - if (info.length.isEmpty && info.downloaded == 0L) - "" - else - s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${info.downloaded}${info.length.map(" / " + _).mkString})" - - val total = url.length + 1 + extra.length - val (url0, extra0) = - if (total >= width) { // or > ? If equal, does it go down 2 lines? - val overflow = total - width + 1 - - val extra0 = - if (extra.length > baseExtraWidth) - extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…" - else - extra - - val total0 = url.length + 1 + extra0.length - val overflow0 = total0 - width + 1 - - val url0 = - if (total0 >= width) - url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…" - else - url - - (url0, extra0) - } else - (url, extra) - - out.write(s"$url0 $extra0\n") - } - - out.flush() - Thread.sleep(refreshInterval) - helper(downloads0.length) - } - - helper(0) - } - } - - t.setDaemon(true) - - def init(): Unit = { - width = TTY.consoleDim("cols") - ansi.clearLine(2) - t.start() - } - - def stop(): Unit = { - q.put(Left(())) - lock.synchronized(()) - } - - private case class Info(downloaded: Long, length: Option[Long]) { - def pct: Option[Double] = length.map(downloaded.toDouble / _) - } - - private val downloads = new ArrayBuffer[String] - private val infos = new ConcurrentHashMap[String, Info] - - private val q = new LinkedBlockingDeque[Either[Unit, Unit]] - def update(): Unit = { - if (q.size() == 0) - q.put(Right(())) - } - - override def downloadingArtifact(url: String): Unit = { - assert(!infos.containsKey(url)) - val prev = infos.putIfAbsent(url, Info(0L, None)) - assert(prev == null) - - downloads.synchronized { - downloads.append(url) - } - - update() - } - override def downloadLength(url: String, length: Long): Unit = { - val info = infos.get(url) - assert(info != null) - val newInfo = info.copy(length = Some(length)) - infos.put(url, newInfo) - - update() - } - override def downloadProgress(url: String, downloaded: Long): Unit = { - val info = infos.get(url) - assert(info != null) - val newInfo = info.copy(downloaded = downloaded) - infos.put(url, newInfo) - - update() - } - override def downloadedArtifact(url: String, success: Boolean): Unit = { - downloads.synchronized { - downloads -= url - } - - val info = infos.remove(url) - assert(info != null) - - update() - } - -} diff --git a/core/js/src/main/scala/coursier/core/compatibility/package.scala b/core/js/src/main/scala/coursier/core/compatibility/package.scala index 9226f6d07..b168d85e1 100644 --- a/core/js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/js/src/main/scala/coursier/core/compatibility/package.scala @@ -46,11 +46,13 @@ package object compatibility { def label = option[String](node0.nodeName) .getOrElse("") - def child = + def children = option[NodeList](node0.childNodes) .map(l => List.tabulate(l.length)(l.item).map(fromNode)) .getOrElse(Nil) + def attributes = ??? + // `exists` instead of `contains`, for scala 2.10 def isText = option[Int](node0.nodeType) diff --git a/core/jvm/src/main/scala/coursier/ResolutionClassLoader.scala b/core/jvm/src/main/scala/coursier/ResolutionClassLoader.scala new file mode 100644 index 000000000..3f852bb9c --- /dev/null +++ b/core/jvm/src/main/scala/coursier/ResolutionClassLoader.scala @@ -0,0 +1,37 @@ +package coursier + +import java.io.File +import java.net.URLClassLoader + +import coursier.util.ClasspathFilter + +class ResolutionClassLoader( + val resolution: Resolution, + val artifacts: Seq[(Dependency, Artifact, File)], + parent: ClassLoader +) extends URLClassLoader( + artifacts.map { case (_, _, f) => f.toURI.toURL }.toArray, + parent +) { + + /** + * Filtered version of this `ClassLoader`, exposing only `dependencies` and their + * their transitive dependencies, and filtering out the other dependencies from + * `resolution` - for `ClassLoader` isolation. + * + * An application launched by `coursier launch -C` has `ResolutionClassLoader` set as its + * context `ClassLoader` (can be obtain with `Thread.currentThread().getContextClassLoader`). + * If it aims at doing `ClassLoader` isolation, exposing only a dependency `dep` to the isolated + * things, `filter(dep)` provides a `ClassLoader` that loaded `dep` and all its transitive + * dependencies through the same loader as the contextual one, but that "exposes" only + * `dep` and its transitive dependencies, nothing more. + */ + def filter(dependencies: Set[Dependency]): ClassLoader = { + val subRes = resolution.subset(dependencies) + val subArtifacts = subRes.dependencyArtifacts.map { case (_, a) => a }.toSet + val subFiles = artifacts.collect { case (_, a, f) if subArtifacts(a) => f } + + new ClasspathFilter(this, subFiles.toSet, exclude = false) + } + +} diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index 153b376e0..3a8f6dc49 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -2,6 +2,8 @@ package coursier.core import coursier.util.Xml +import scala.xml.{ Attribute, MetaData, Null } + package object compatibility { implicit class RichChar(val c: Char) extends AnyVal { @@ -16,11 +18,32 @@ package object compatibility { def fromNode(node: scala.xml.Node): Xml.Node = new Xml.Node { + lazy val attributes = { + def helper(m: MetaData): Stream[(String, String, String)] = + m match { + case Null => Stream.empty + case attr => + val pre = attr match { + case a: Attribute => Option(node.getNamespace(a.pre)).getOrElse("") + case _ => "" + } + + val value = attr.value.collect { + case scala.xml.Text(t) => t + }.mkString("") + + (pre, attr.key, value) #:: helper(m.next) + } + + helper(node.attributes).toVector + } def label = node.label - def child = node.child.map(fromNode) + def children = node.child.map(fromNode) def isText = node match { case _: scala.xml.Text => true; case _ => false } def textContent = node.text def isElement = node match { case _: scala.xml.Elem => true; case _ => false } + + override def toString = node.toString } parse.right diff --git a/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala b/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala index 4f7cafe4a..7272a2723 100644 --- a/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala +++ b/core/jvm/src/main/scala/coursier/util/ClasspathFilter.scala @@ -84,9 +84,23 @@ class ClasspathFilter(parent: ClassLoader, classpath: Set[File], exclude: Boolea override def loadClass(className: String, resolve: Boolean): Class[_] = { - val c = super.loadClass(className, resolve) - if (fromClasspath(c)) c - else throw new ClassNotFoundException(className) + val c = + try super.loadClass(className, resolve) + catch { + case e: LinkageError => + // Happens when trying to derive a shapeless.Generic + // from an Ammonite session launched like + // ./coursier launch com.lihaoyi:ammonite-repl_2.11.7:0.5.2 + // For className == "shapeless.GenericMacros", + // the super.loadClass above - which would be filtered out below anyway, + // raises a NoClassDefFoundError. + null + } + + if (c != null && fromClasspath(c)) + c + else + throw new ClassNotFoundException(className) } override def getResource(name: String): URL = { diff --git a/core/shared/src/main/scala/coursier/Fetch.scala b/core/shared/src/main/scala/coursier/Fetch.scala new file mode 100644 index 000000000..65d393ad3 --- /dev/null +++ b/core/shared/src/main/scala/coursier/Fetch.scala @@ -0,0 +1,104 @@ +package coursier + +import coursier.maven.MavenSource + +import scalaz._ + +object Fetch { + + type Content[F[_]] = Artifact => EitherT[F, String, String] + + + type MD = Seq[( + (Module, String), + Seq[String] \/ (Artifact.Source, Project) + )] + + type Metadata[F[_]] = Seq[(Module, String)] => F[MD] + + /** + * Try to find `module` among `repositories`. + * + * Look at `repositories` from the left, one-by-one, and stop at first success. + * Else, return all errors, in the same order. + * + * The `version` field of the returned `Project` in case of success may not be + * equal to the provided one, in case the latter is not a specific + * version (e.g. version interval). Which version get chosen depends on + * the repository implementation. + */ + def find[F[_]]( + repositories: Seq[Repository], + module: Module, + version: String, + fetch: Content[F] + )(implicit + F: Monad[F] + ): EitherT[F, Seq[String], (Artifact.Source, Project)] = { + + val lookups = repositories + .map(repo => repo -> repo.find(module, version, fetch).run) + + val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { + case (acc, (repo, eitherProjTask)) => + val looseModuleValidation = repo match { + case m: MavenRepository => m.sbtAttrStub // that sucks so much + case _ => false + } + val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module + F.bind(acc) { + case -\/(errors) => + F.map(eitherProjTask)(_.flatMap{case (source, project) => + val projModule = + if (looseModuleValidation) + project.module.copy(attributes = Map.empty) + else + project.module + if (projModule == moduleCmp) \/-((source, project)) + else -\/(s"Wrong module returned (expected: $moduleCmp, got: ${project.module})") + }.leftMap(error => error +: errors)) + + case res @ \/-(_) => + F.point(res) + } + } + + EitherT(F.map(task)(_.leftMap(_.reverse))) + .map {case x @ (source, proj) => + val looseModuleValidation = source match { + case m: MavenSource => m.sbtAttrStub // omfg + case _ => false + } + val projModule = + if (looseModuleValidation) + proj.module.copy(attributes = Map.empty) + else + proj.module + val moduleCmp = if (looseModuleValidation) module.copy(attributes = Map.empty) else module + assert(projModule == moduleCmp) + x + } + } + + def apply[F[_]]( + repositories: Seq[core.Repository], + fetch: Content[F], + extra: Content[F]* + )(implicit + F: Nondeterminism[F] + ): Metadata[F] = { + + modVers => + F.map( + F.gatherUnordered( + modVers.map { case (module, version) => + def get(fetch: Content[F]) = + find(repositories, module, version, fetch) + F.map((get(fetch) /: extra)(_ orElse get(_)) + .run)((module, version) -> _) + } + ) + )(_.toSeq) + } + +} \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 130b48092..de4604840 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -8,12 +8,11 @@ package coursier.core * between them. * * Using the same terminology as Ivy. - * - * Ivy attributes would land here, if support for it is added. */ case class Module( organization: String, - name: String + name: String, + attributes: Map[String, String] ) { def trim: Module = copy( @@ -21,10 +20,15 @@ case class Module( name = name.trim ) - override def toString = s"$organization:$name" -} + private def attributesStr = attributes.toSeq + .sortBy { case (k, _) => k } + .map { case (k, v) => s"$k=$v" } + .mkString(";") -sealed abstract class Scope(val name: String) + override def toString = + s"$organization:$name" + + (if (attributes.nonEmpty) s";$attributesStr" else "") +} /** * Dependencies with the same @module will typically see their @version-s merged. @@ -35,53 +39,88 @@ sealed abstract class Scope(val name: String) case class Dependency( module: Module, version: String, - scope: Scope, - attributes: Attributes, + configuration: String, exclusions: Set[(String, String)], + + // Maven-specific + attributes: Attributes, optional: Boolean ) { def moduleVersion = (module, version) } +// Maven-specific case class Attributes( `type`: String, classifier: String -) +) { + def publication(name: String, ext: String): Publication = + Publication(name, `type`, ext, classifier) +} case class Project( module: Module, version: String, - dependencies: Seq[Dependency], + // First String is configuration (scope for Maven) + dependencies: Seq[(String, Dependency)], + // For Maven, this is the standard scopes as an Ivy configuration + configurations: Map[String, Seq[String]], + + // Maven-specific parent: Option[(Module, String)], - dependencyManagement: Seq[Dependency], - properties: Map[String, String], + dependencyManagement: Seq[(String, Dependency)], + properties: Seq[(String, String)], profiles: Seq[Profile], versions: Option[Versions], - snapshotVersioning: Option[SnapshotVersioning] + snapshotVersioning: Option[SnapshotVersioning], + + // Ivy-specific + // First String is configuration + publications: Seq[(String, Publication)], + + // Extra infos, not used during resolution + info: Info ) { def moduleVersion = (module, version) + + /** All configurations that each configuration extends, including the ones it extends transitively */ + lazy val allConfigurations: Map[String, Set[String]] = + Orders.allConfigurations(configurations) } -object Scope { - case object Compile extends Scope("compile") - case object Runtime extends Scope("runtime") - case object Test extends Scope("test") - case object Provided extends Scope("provided") - case object Import extends Scope("import") - case class Other(override val name: String) extends Scope(name) +/** Extra project info, not used during resolution */ +case class Info( + description: String, + homePage: String, + licenses: Seq[(String, Option[String])], + developers: Seq[Info.Developer], + publication: Option[Versions.DateTime] +) + +object Info { + case class Developer( + id: String, + name: String, + url: String + ) + + val empty = Info("", "", Nil, Nil, None) } +// Maven-specific case class Activation(properties: Seq[(String, Option[String])]) +// Maven-specific case class Profile( id: String, activeByDefault: Option[Boolean], activation: Activation, - dependencies: Seq[Dependency], - dependencyManagement: Seq[Dependency], + dependencies: Seq[(String, Dependency)], + dependencyManagement: Seq[(String, Dependency)], properties: Map[String, String] ) +// Maven-specific case class Versions( latest: String, release: String, @@ -100,6 +139,7 @@ object Versions { ) } +// Maven-specific case class SnapshotVersion( classifier: String, extension: String, @@ -107,6 +147,7 @@ case class SnapshotVersion( updated: Option[Versions.DateTime] ) +// Maven-specific case class SnapshotVersioning( module: Module, version: String, @@ -119,15 +160,40 @@ case class SnapshotVersioning( snapshotVersions: Seq[SnapshotVersion] ) +// Ivy-specific +case class Publication( + name: String, + `type`: String, + ext: String, + classifier: String +) { + def attributes: Attributes = Attributes(`type`, classifier) +} + case class Artifact( url: String, checksumUrls: Map[String, String], extra: Map[String, Artifact], - attributes: Attributes + attributes: Attributes, + changing: Boolean ) object Artifact { trait Source { - def artifacts(dependency: Dependency, project: Project): Seq[Artifact] + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ): Seq[Artifact] + } + + object Source { + val empty: Source = new Source { + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ): Seq[Artifact] = Nil + } } } diff --git a/core/shared/src/main/scala/coursier/core/Orders.scala b/core/shared/src/main/scala/coursier/core/Orders.scala index dd5c7a28a..15af6bfe2 100644 --- a/core/shared/src/main/scala/coursier/core/Orders.scala +++ b/core/shared/src/main/scala/coursier/core/Orders.scala @@ -2,39 +2,61 @@ package coursier.core object Orders { - /** Minimal ad-hoc partial order */ - trait PartialOrder[A] { - /** - * x < y: Some(neg. integer) - * x == y: Some(0) - * x > y: Some(pos. integer) - * x, y not related: None - */ - def cmp(x: A, y: A): Option[Int] + trait PartialOrdering[T] extends scala.math.PartialOrdering[T] { + def lteq(x: T, y: T): Boolean = + tryCompare(x, y) + .exists(_ <= 0) + } + + /** All configurations that each configuration extends, including the ones it extends transitively */ + def allConfigurations(configurations: Map[String, Seq[String]]): Map[String, Set[String]] = { + def allParents(config: String): Set[String] = { + def helper(configs: Set[String], acc: Set[String]): Set[String] = + if (configs.isEmpty) + acc + else if (configs.exists(acc)) + helper(configs -- acc, acc) + else if (configs.exists(!configurations.contains(_))) { + val (remaining, notFound) = configs.partition(configurations.contains) + helper(remaining, acc ++ notFound) + } else { + val extraConfigs = configs.flatMap(configurations) + helper(extraConfigs, acc ++ configs) + } + + helper(Set(config), Set.empty) + } + + configurations + .keys + .toList + .map(config => config -> (allParents(config) - config)) + .toMap } /** * Only relations: * Compile < Runtime < Test */ - implicit val mavenScopePartialOrder: PartialOrder[Scope] = - new PartialOrder[Scope] { - val higher = Map[Scope, Set[Scope]]( - Scope.Compile -> Set(Scope.Runtime, Scope.Test), - Scope.Runtime -> Set(Scope.Test) - ) + def configurationPartialOrder(configurations: Map[String, Seq[String]]): PartialOrdering[String] = + new PartialOrdering[String] { + val allParentsMap = allConfigurations(configurations) - def cmp(x: Scope, y: Scope) = - if (x == y) Some(0) - else if (higher.get(x).exists(_(y))) Some(-1) - else if (higher.get(y).exists(_(x))) Some(1) - else None + def tryCompare(x: String, y: String) = + if (x == y) + Some(0) + else if (allParentsMap.get(x).exists(_(y))) + Some(-1) + else if (allParentsMap.get(y).exists(_(x))) + Some(1) + else + None } /** Non-optional < optional */ - implicit val optionalPartialOrder: PartialOrder[Boolean] = - new PartialOrder[Boolean] { - def cmp(x: Boolean, y: Boolean) = + val optionalPartialOrder: PartialOrdering[Boolean] = + new PartialOrdering[Boolean] { + def tryCompare(x: Boolean, y: Boolean) = Some( if (x == y) 0 else if (x) 1 @@ -51,8 +73,8 @@ object Orders { * * In particular, no exclusions <= anything <= Set(("*", "*")) */ - implicit val exclusionsPartialOrder: PartialOrder[Set[(String, String)]] = - new PartialOrder[Set[(String, String)]] { + val exclusionsPartialOrder: PartialOrdering[Set[(String, String)]] = + new PartialOrdering[Set[(String, String)]] { def boolCmp(a: Boolean, b: Boolean) = (a, b) match { case (true, true) => Some(0) case (true, false) => Some(1) @@ -60,7 +82,7 @@ object Orders { case (false, false) => None } - def cmp(x: Set[(String, String)], y: Set[(String, String)]) = { + def tryCompare(x: Set[(String, String)], y: Set[(String, String)]) = { val (xAll, xExcludeByOrg1, xExcludeByName1, xRemaining0) = Exclusions.partition(x) val (yAll, yExcludeByOrg1, yExcludeByName1, yRemaining0) = Exclusions.partition(y) @@ -98,23 +120,44 @@ object Orders { } } + private def fallbackConfigIfNecessary(dep: Dependency, configs: Set[String]): Dependency = + Parse.withFallbackConfig(dep.configuration) match { + case Some((main, fallback)) => + val config0 = + if (configs(main)) + main + else if (configs(fallback)) + fallback + else + dep.configuration + + dep.copy(configuration = config0) + case _ => + dep + } + /** * Assume all dependencies have same `module`, `version`, and `artifact`; see `minDependencies` * if they don't. */ - def minDependenciesUnsafe(dependencies: Set[Dependency]): Set[Dependency] = { + def minDependenciesUnsafe( + dependencies: Set[Dependency], + configs: Map[String, Seq[String]] + ): Set[Dependency] = { + val availableConfigs = configs.keySet val groupedDependencies = dependencies - .groupBy(dep => (dep.optional, dep.scope)) + .map(fallbackConfigIfNecessary(_, availableConfigs)) + .groupBy(dep => (dep.optional, dep.configuration)) .mapValues(deps => deps.head.copy(exclusions = deps.foldLeft(Exclusions.one)((acc, dep) => Exclusions.meet(acc, dep.exclusions)))) .toList val remove = for { List(((xOpt, xScope), xDep), ((yOpt, yScope), yDep)) <- groupedDependencies.combinations(2) - optCmp <- optionalPartialOrder.cmp(xOpt, yOpt).iterator - scopeCmp <- mavenScopePartialOrder.cmp(xScope, yScope).iterator + optCmp <- optionalPartialOrder.tryCompare(xOpt, yOpt).iterator + scopeCmp <- configurationPartialOrder(configs).tryCompare(xScope, yScope).iterator if optCmp*scopeCmp >= 0 - exclCmp <- exclusionsPartialOrder.cmp(xDep.exclusions, yDep.exclusions).iterator + exclCmp <- exclusionsPartialOrder.tryCompare(xDep.exclusions, yDep.exclusions).iterator if optCmp*exclCmp >= 0 if scopeCmp*exclCmp >= 0 xIsMin = optCmp < 0 || scopeCmp < 0 || exclCmp < 0 @@ -130,10 +173,13 @@ object Orders { * * The returned set brings exactly the same things as `dependencies`, with no redundancy. */ - def minDependencies(dependencies: Set[Dependency]): Set[Dependency] = { + def minDependencies( + dependencies: Set[Dependency], + configs: ((Module, String)) => Map[String, Seq[String]] + ): Set[Dependency] = { dependencies - .groupBy(_.copy(scope = Scope.Other(""), exclusions = Set.empty, optional = false)) - .mapValues(minDependenciesUnsafe) + .groupBy(_.copy(configuration = "", exclusions = Set.empty, optional = false)) + .mapValues(deps => minDependenciesUnsafe(deps, configs(deps.head.moduleVersion))) .valuesIterator .fold(Set.empty)(_ ++ _) } diff --git a/core/shared/src/main/scala/coursier/core/Parse.scala b/core/shared/src/main/scala/coursier/core/Parse.scala index 4d89c35ea..a813b26f1 100644 --- a/core/shared/src/main/scala/coursier/core/Parse.scala +++ b/core/shared/src/main/scala/coursier/core/Parse.scala @@ -1,18 +1,10 @@ package coursier.core +import java.util.regex.Pattern.quote import coursier.core.compatibility._ object Parse { - def scope(s: String): Scope = s match { - case "compile" => Scope.Compile - case "runtime" => Scope.Runtime - case "test" => Scope.Test - case "provided" => Scope.Provided - case "import" => Scope.Import - case other => Scope.Other(other) - } - def version(s: String): Option[Version] = { if (s.isEmpty || s.exists(c => c != '.' && c != '-' && c != '_' && !c.letterOrDigit)) None else Some(Version(s)) @@ -40,4 +32,20 @@ object Parse { .orElse(versionInterval(s).map(VersionConstraint.Interval)) } + val fallbackConfigRegex = { + val noPar = "([^" + quote("()") + "]*)" + "^" + noPar + quote("(") + noPar + quote(")") + "$" + }.r + + def withFallbackConfig(config: String): Option[(String, String)] = + Parse.fallbackConfigRegex.findAllMatchIn(config).toSeq match { + case Seq(m) => + assert(m.groupCount == 2) + val main = config.substring(m.start(1), m.end(1)) + val fallback = config.substring(m.start(2), m.end(2)) + Some((main, fallback)) + case _ => + None + } + } diff --git a/core/shared/src/main/scala/coursier/core/Repository.scala b/core/shared/src/main/scala/coursier/core/Repository.scala index 69bcfa457..297f95f9a 100644 --- a/core/shared/src/main/scala/coursier/core/Repository.scala +++ b/core/shared/src/main/scala/coursier/core/Repository.scala @@ -1,5 +1,7 @@ package coursier.core +import coursier.Fetch + import scala.language.higherKinds import scalaz._ @@ -10,7 +12,7 @@ trait Repository { def find[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, (Artifact.Source, Project)] @@ -18,52 +20,6 @@ trait Repository { object Repository { - type Fetch[F[_]] = Artifact => EitherT[F, String, String] - - /** - * Try to find `module` among `repositories`. - * - * Look at `repositories` from the left, one-by-one, and stop at first success. - * Else, return all errors, in the same order. - * - * The `version` field of the returned `Project` in case of success may not be - * equal to the provided one, in case the latter is not a specific - * version (e.g. version interval). Which version get chosen depends on - * the repository implementation. - */ - def find[F[_]]( - repositories: Seq[Repository], - module: Module, - version: String, - fetch: Repository.Fetch[F] - )(implicit - F: Monad[F] - ): EitherT[F, Seq[String], (Artifact.Source, Project)] = { - - val lookups = repositories - .map(repo => repo -> repo.find(module, version, fetch).run) - - val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { - case (acc, (repo, eitherProjTask)) => - F.bind(acc) { - case -\/(errors) => - F.map(eitherProjTask)(_.flatMap{case (source, project) => - if (project.module == module) \/-((source, project)) - else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") - }.leftMap(error => error +: errors)) - - case res @ \/-(_) => - F.point(res) - } - } - - EitherT(F.map(task)(_.leftMap(_.reverse))) - .map {case x @ (_, proj) => - assert(proj.module == module) - x - } - } - implicit class ArtifactExtensions(val underlying: Artifact) extends AnyVal { def withDefaultChecksums: Artifact = underlying.copy(checksumUrls = underlying.checksumUrls ++ Seq( @@ -73,20 +29,15 @@ object Repository { def withDefaultSignature: Artifact = underlying.copy(extra = underlying.extra ++ Seq( "sig" -> - Artifact(underlying.url + ".asc", Map.empty, Map.empty, Attributes("asc", "")) + Artifact( + underlying.url + ".asc", + Map.empty, + Map.empty, + Attributes("asc", ""), + changing = underlying.changing + ) .withDefaultChecksums )) - def withJavadocSources: Artifact = { - val base = underlying.url.stripSuffix(".jar") - underlying.copy(extra = underlying.extra ++ Seq( - "sources" -> Artifact(base + "-sources.jar", Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? - .withDefaultChecksums - .withDefaultSignature, - "javadoc" -> Artifact(base + "-javadoc.jar", Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above - .withDefaultChecksums - .withDefaultSignature - )) - } } } diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index fa4203bdd..bd534e015 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -37,39 +37,33 @@ object Resolution { (dep.module.organization, dep.module.name, dep.attributes.`type`) def add( - dict: Map[Key, Dependency], - dep: Dependency - ): Map[Key, Dependency] = { + dict: Map[Key, (String, Dependency)], + item: (String, Dependency) + ): Map[Key, (String, Dependency)] = { - val key0 = key(dep) + val key0 = key(item._2) if (dict.contains(key0)) dict else - dict + (key0 -> dep) + dict + (key0 -> item) } def addSeq( - dict: Map[Key, Dependency], - deps: Seq[Dependency] - ): Map[Key, Dependency] = + dict: Map[Key, (String, Dependency)], + deps: Seq[(String, Dependency)] + ): Map[Key, (String, Dependency)] = (dict /: deps)(add) } - def mergeProperties( - dict: Map[String, String], - other: Map[String, String] - ): Map[String, String] = - dict ++ other.filterKeys(!dict.contains(_)) - - def addDependencies(deps: Seq[Seq[Dependency]]): Seq[Dependency] = { + def addDependencies(deps: Seq[Seq[(String, Dependency)]]): Seq[(String, Dependency)] = { val res = - (deps :\ (Set.empty[DepMgmt.Key], Seq.empty[Dependency])) { + (deps :\ (Set.empty[DepMgmt.Key], Seq.empty[(String, Dependency)])) { case (deps0, (set, acc)) => val deps = deps0 - .filter(dep => !set(DepMgmt.key(dep))) + .filter{case (_, dep) => !set(DepMgmt.key(dep))} - (set ++ deps.map(DepMgmt.key), acc ++ deps) + (set ++ deps.map{case (_, dep) => DepMgmt.key(dep)}, acc ++ deps) } res._2 @@ -79,49 +73,59 @@ object Resolution { quote("${") + "([a-zA-Z0-9-.]*)" + quote("}") ).r + def substituteProps(s: String, properties: Map[String, String]) = { + val matches = propRegex + .findAllMatchIn(s) + .toList + .reverse + + if (matches.isEmpty) s + else { + val output = + (new StringBuilder(s) /: matches) { (b, m) => + properties + .get(m.group(1)) + .fold(b)(b.replace(m.start, m.end, _)) + } + + output.result() + } + } + + def propertiesMap(props: Seq[(String, String)]): Map[String, String] = + props.foldLeft(Map.empty[String, String]) { + case (acc, (k, v0)) => + val v = substituteProps(v0, acc) + acc + (k -> v) + } + /** * Substitutes `properties` in `dependencies`. */ def withProperties( - dependencies: Seq[Dependency], + dependencies: Seq[(String, Dependency)], properties: Map[String, String] - ): Seq[Dependency] = { + ): Seq[(String, Dependency)] = { - def substituteProps(s: String) = { - val matches = propRegex - .findAllMatchIn(s) - .toList - .reverse - - if (matches.isEmpty) s - else { - val output = - (new StringBuilder(s) /: matches) { (b, m) => - properties - .get(m.group(1)) - .fold(b)(b.replace(m.start, m.end, _)) - } - - output.result() - } - } + def substituteProps0(s: String) = + substituteProps(s, properties) dependencies - .map{ dep => - dep.copy( + .map {case (config, dep) => + substituteProps0(config) -> dep.copy( module = dep.module.copy( - organization = substituteProps(dep.module.organization), - name = substituteProps(dep.module.name) + organization = substituteProps0(dep.module.organization), + name = substituteProps0(dep.module.name) ), - version = substituteProps(dep.version), + version = substituteProps0(dep.version), attributes = dep.attributes.copy( - `type` = substituteProps(dep.attributes.`type`), - classifier = substituteProps(dep.attributes.classifier) + `type` = substituteProps0(dep.attributes.`type`), + classifier = substituteProps0(dep.attributes.classifier) ), - scope = Parse.scope(substituteProps(dep.scope.name)), + configuration = substituteProps0(dep.configuration), exclusions = dep.exclusions .map{case (org, name) => - (substituteProps(org), substituteProps(name)) + (substituteProps0(org), substituteProps0(name)) } // FIXME The content of the optional tag may also be a property in // the original POM. Maybe not parse it that earlier? @@ -165,33 +169,35 @@ object Resolution { def merge( dependencies: TraversableOnce[Dependency], forceVersions: Map[Module, String] - ): (Seq[Dependency], Seq[Dependency]) = { + ): (Seq[Dependency], Seq[Dependency], Map[Module, String]) = { val mergedByModVer = dependencies .toList .groupBy(dep => dep.module) .map { case (module, deps) => module -> { - forceVersions.get(module) match { + val (versionOpt, updatedDeps) = forceVersions.get(module) match { case None => - if (deps.lengthCompare(1) == 0) \/-(deps) + if (deps.lengthCompare(1) == 0) (Some(deps.head.version), \/-(deps)) else { val versions = deps .map(_.version) .distinct val versionOpt = mergeVersions(versions) - versionOpt match { + (versionOpt, versionOpt match { case Some(version) => \/-(deps.map(dep => dep.copy(version = version))) case None => -\/(deps) - } + }) } case Some(forcedVersion) => - \/-(deps.map(dep => dep.copy(version = forcedVersion))) + (Some(forcedVersion), \/-(deps.map(dep => dep.copy(version = forcedVersion)))) } + + (updatedDeps, versionOpt) } } @@ -201,33 +207,16 @@ object Resolution { ( merged - .collect{case -\/(dep) => dep} + .collect { case (-\/(dep), _) => dep } .flatten, merged - .collect{case \/-(dep) => dep} - .flatten + .collect { case (\/-(dep), _) => dep } + .flatten, + mergedByModVer + .collect { case (mod, (_, Some(ver))) => mod -> ver } ) } - /** - * If one of our dependency has scope `base`, and a transitive - * dependency of it has scope `transitive`, return the scope of - * the latter for us, if any. If empty, means the transitive dependency - * should not be considered a dependency for us. - * - * See https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Scope. - */ - def resolveScope( - base: Scope, - transitive: Scope - ): Option[Scope] = - (base, transitive) match { - case (other, Scope.Compile) => Some(other) - case (Scope.Compile, Scope.Runtime) => Some(Scope.Runtime) - case (other, Scope.Runtime) => Some(other) - case _ => None - } - /** * Applies `dependencyManagement` to `dependencies`. * @@ -235,36 +224,39 @@ object Resolution { * `dependencyManagement`. */ def depsWithDependencyManagement( - dependencies: Seq[Dependency], - dependencyManagement: Seq[Dependency] - ): Seq[Dependency] = { + dependencies: Seq[(String, Dependency)], + dependencyManagement: Seq[(String, Dependency)] + ): Seq[(String, Dependency)] = { // See http://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Dependency_Management lazy val dict = DepMgmt.addSeq(Map.empty, dependencyManagement) dependencies - .map { dep0 => + .map {case (config0, dep0) => + var config = config0 var dep = dep0 - for (mgmtDep <- dict.get(DepMgmt.key(dep0))) { + for ((mgmtConfig, mgmtDep) <- dict.get(DepMgmt.key(dep0))) { if (dep.version.isEmpty) dep = dep.copy(version = mgmtDep.version) - if (dep.scope.name.isEmpty) - dep = dep.copy(scope = mgmtDep.scope) + if (config.isEmpty) + config = mgmtConfig if (dep.exclusions.isEmpty) dep = dep.copy(exclusions = mgmtDep.exclusions) } - dep + (config, dep) } } - def withDefaultScope(dep: Dependency): Dependency = - if (dep.scope.name.isEmpty) - dep.copy(scope = Scope.Compile) + val defaultConfiguration = "compile" + + def withDefaultConfig(dep: Dependency): Dependency = + if (dep.configuration.isEmpty) + dep.copy(configuration = defaultConfiguration) else dep @@ -272,19 +264,48 @@ object Resolution { * Filters `dependencies` with `exclusions`. */ def withExclusions( - dependencies: Seq[Dependency], + dependencies: Seq[(String, Dependency)], exclusions: Set[(String, String)] - ): Seq[Dependency] = { + ): Seq[(String, Dependency)] = { val filter = Exclusions(exclusions) dependencies - .filter(dep => filter(dep.module.organization, dep.module.name)) - .map(dep => - dep.copy( + .filter{case (_, dep) => filter(dep.module.organization, dep.module.name) } + .map{case (config, dep) => + config -> dep.copy( exclusions = Exclusions.minimize(dep.exclusions ++ exclusions) ) - ) + } + } + + def withParentConfigurations(config: String, configurations: Map[String, Seq[String]]): Set[String] = { + @tailrec + def helper(configs: Set[String], acc: Set[String]): Set[String] = + if (configs.isEmpty) + acc + else if (configs.exists(acc)) + helper(configs -- acc, acc) + else if (configs.exists(!configurations.contains(_))) { + val (remaining, notFound) = configs.partition(configurations.contains) + helper(remaining, acc ++ notFound) + } else { + val extraConfigs = configs.flatMap(configurations) + helper(extraConfigs, acc ++ configs) + } + + val config0 = Parse.withFallbackConfig(config) match { + case Some((main, fallback)) => + if (configurations.contains(main)) + main + else if (configurations.contains(fallback)) + fallback + else + main + case None => config + } + + helper(Set(config0), Set.empty) } /** @@ -303,38 +324,48 @@ object Resolution { // come from parents or dependency management. This may not be // the right thing to do. - val properties = mergeProperties( - project.properties, - Map( - "project.groupId" -> project.module.organization, - "project.artifactId" -> project.module.name, - "project.version" -> project.version - ) + // FIXME The extra properties should only be added for Maven projects, not Ivy ones + val properties0 = Seq( + // some artifacts seem to require these (e.g. org.jmock:jmock-legacy:2.5.1) + // although I can find no mention of them in any manual / spec + "pom.groupId" -> project.module.organization, + "pom.artifactId" -> project.module.name, + "pom.version" -> project.version + ) ++ project.properties ++ Seq( + "project.groupId" -> project.module.organization, + "project.artifactId" -> project.module.name, + "project.version" -> project.version ) - val deps = - withExclusions( - depsWithDependencyManagement( - // Important: properties have to be applied to both, - // so that dep mgmt can be matched properly - // Tested with org.ow2.asm:asm-commons:5.0.2 in CentralTests - withProperties(project.dependencies, properties), - withProperties(project.dependencyManagement, properties) - ), - from.exclusions - ) - .map(withDefaultScope) + val properties = propertiesMap(properties0) - deps - .flatMap { trDep => - resolveScope(from.scope, trDep.scope) - .map(scope => - trDep.copy( - scope = scope, - optional = trDep.optional || from.optional - ) - ) - } + val configurations = withParentConfigurations(from.configuration, project.configurations) + + withExclusions( + depsWithDependencyManagement( + // Important: properties have to be applied to both, + // so that dep mgmt can be matched properly + // Tested with org.ow2.asm:asm-commons:5.0.2 in CentralTests + withProperties(project.dependencies, properties), + withProperties(project.dependencyManagement, properties) + ), + from.exclusions + ) + .map{ + case (config, dep) => + (if (config.isEmpty) defaultConfiguration else config) -> { + if (dep.configuration.isEmpty) + dep.copy(configuration = defaultConfiguration) + else + dep + } + } + .collect{case (config, dep) if configurations(config) => + if (from.optional) + dep.copy(optional = true) + else + dep + } } /** @@ -432,13 +463,13 @@ case class Resolution( * May contain dependencies added in previous iterations, but no more * required. These are filtered below, see `newDependencies`. * - * Returns a tuple made of the conflicting dependencies, and all - * the dependencies. + * Returns a tuple made of the conflicting dependencies, all + * the dependencies, and the retained version of each module. */ - def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency]) = + def nextDependenciesAndConflicts: (Seq[Dependency], Seq[Dependency], Map[Module, String]) = // TODO Provide the modules whose version was forced by dependency overrides too merge( - rootDependencies.map(withDefaultScope) ++ dependencies ++ transitiveDependencies, + rootDependencies.map(withDefaultConfig) ++ dependencies ++ transitiveDependencies, forceVersions ) @@ -461,7 +492,7 @@ case class Resolution( */ def isDone: Boolean = { def isFixPoint = { - val (nextConflicts, _) = nextDependenciesAndConflicts + val (nextConflicts, _, _) = nextDependenciesAndConflicts dependencies == (newDependencies ++ nextConflicts) && conflicts == nextConflicts.toSet @@ -480,7 +511,7 @@ case class Resolution( * The versions of all the dependencies returned are erased (emptied). */ def reverseDependencies: Map[Dependency, Vector[Dependency]] = { - val (updatedConflicts, updatedDeps) = nextDependenciesAndConflicts + val (updatedConflicts, updatedDeps, _) = nextDependenciesAndConflicts val trDepsSeq = for { @@ -507,7 +538,7 @@ case class Resolution( */ def remainingDependencies: Set[Dependency] = { val rootDependencies0 = rootDependencies - .map(withDefaultScope) + .map(withDefaultConfig) .map(eraseVersion) @tailrec @@ -549,7 +580,7 @@ case class Resolution( } private def nextNoMissingUnsafe: Resolution = { - val (newConflicts, _) = nextDependenciesAndConflicts + val (newConflicts, _, _) = nextDependenciesAndConflicts copy( dependencies = newDependencies ++ newConflicts, @@ -583,24 +614,31 @@ case class Resolution( project: Project ): Set[ModuleVersion] = { - val approxProperties = + val approxProperties0 = project.parent .flatMap(projectCache.get) .map(_._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) + .fold(project.properties)(project.properties ++ _) + + val approxProperties = propertiesMap(approxProperties0) ++ Seq( + "project.groupId" -> project.module.organization, + "project.artifactId" -> project.module.name, + "project.version" -> project.version + ) val profileDependencies = profiles( project, approxProperties, profileActivation getOrElse defaultProfileActivation - ).flatMap(_.dependencies) + ).flatMap(p => p.dependencies ++ p.dependencyManagement) - val modules = - (project.dependencies ++ profileDependencies) - .collect{ - case dep if dep.scope == Scope.Import => dep.moduleVersion - } + val modules = withProperties( + project.dependencies ++ project.dependencyManagement ++ profileDependencies, + approxProperties + ).collect { + case ("import", dep) => dep.moduleVersion + } modules.toSet ++ project.parent } @@ -657,11 +695,21 @@ case class Resolution( */ def withDependencyManagement(project: Project): Project = { - val approxProperties = + // A bit fragile, but seems to work + // TODO Add non regression test for the touchy org.glassfish.jersey.core:jersey-client:2.19 + // (for the way it uses org.glassfish.hk2:hk2-bom,2.4.0-b25) + + val approxProperties0 = project.parent .filter(projectCache.contains) - .map(projectCache(_)._2.properties) - .fold(project.properties)(mergeProperties(project.properties, _)) + .map(projectCache(_)._2.properties.toMap) + .fold(project.properties)(project.properties ++ _) + + val approxProperties = propertiesMap(approxProperties0) ++ Seq( + "project.groupId" -> project.module.organization, + "project.artifactId" -> project.module.name, + "project.version" -> project.version + ) val profiles0 = profiles( project, @@ -670,20 +718,29 @@ case class Resolution( ) val dependencies0 = addDependencies( - project.dependencies +: profiles0.map(_.dependencies) + (project.dependencies +: profiles0.map(_.dependencies)).map(withProperties(_, approxProperties)) + ) + val dependenciesMgmt0 = addDependencies( + (project.dependencyManagement +: profiles0.map(_.dependencyManagement)).map(withProperties(_, approxProperties)) ) val properties0 = (project.properties /: profiles0) { (acc, p) => - mergeProperties(acc, p.properties) + acc ++ p.properties } - val deps = ( + val deps0 = ( dependencies0 - .collect { case dep if dep.scope == Scope.Import => + .collect { case ("import", dep) => + dep.moduleVersion + } ++ + dependenciesMgmt0 + .collect { case ("import", dep) => dep.moduleVersion } ++ project.parent - ).filter(projectCache.contains) + ) + + val deps = deps0.filter(projectCache.contains) val projs = deps .map(projectCache(_)._2) @@ -693,41 +750,75 @@ case class Resolution( profiles0.map(_.dependencyManagement) ++ projs.map(_.dependencyManagement) ) - ).foldLeft(Map.empty[DepMgmt.Key, Dependency])(DepMgmt.addSeq) + ) + .map(withProperties(_, approxProperties)) + .foldLeft(Map.empty[DepMgmt.Key, (String, Dependency)])(DepMgmt.addSeq) val depsSet = deps.toSet project.copy( dependencies = dependencies0 - .filterNot(dep => - dep.scope == Scope.Import && depsSet(dep.moduleVersion) - ) ++ + .filterNot{case (config, dep) => + config == "import" && depsSet(dep.moduleVersion) + } ++ project.parent .filter(projectCache.contains) .toSeq .flatMap(projectCache(_)._2.dependencies), - dependencyManagement = depMgmt.values.toSeq, + dependencyManagement = depMgmt.values.toSeq + .filterNot{case (config, dep) => + config == "import" && depsSet(dep.moduleVersion) + }, properties = project.parent .filter(projectCache.contains) .map(projectCache(_)._2.properties) - .fold(properties0)(mergeProperties(properties0, _)) + .fold(properties0)(properties0 ++ _) ) } def minDependencies: Set[Dependency] = - Orders.minDependencies(dependencies) + Orders.minDependencies( + dependencies, + dep => + projectCache + .get(dep) + .map(_._2.configurations) + .getOrElse(Map.empty) + ) - def artifacts: Seq[Artifact] = + private def artifacts0(overrideClassifiers: Option[Seq[String]]): Seq[Artifact] = for { dep <- minDependencies.toSeq (source, proj) <- projectCache .get(dep.moduleVersion) .toSeq artifact <- source - .artifacts(dep, proj) + .artifacts(dep, proj, overrideClassifiers) } yield artifact + def classifiersArtifacts(classifiers: Seq[String]): Seq[Artifact] = + artifacts0(Some(classifiers)) + + def artifacts: Seq[Artifact] = + artifacts0(None) + + private def dependencyArtifacts0(overrideClassifiers: Option[Seq[String]]): Seq[(Dependency, Artifact)] = + for { + dep <- minDependencies.toSeq + (source, proj) <- projectCache + .get(dep.moduleVersion) + .toSeq + artifact <- source + .artifacts(dep, proj, overrideClassifiers) + } yield dep -> artifact + + def dependencyArtifacts: Seq[(Dependency, Artifact)] = + dependencyArtifacts0(None) + + def dependencyClassifiersArtifacts(classifiers: Seq[String]): Seq[(Dependency, Artifact)] = + dependencyArtifacts0(Some(classifiers)) + def errors: Seq[(Dependency, Seq[String])] = for { dep <- dependencies.toSeq @@ -735,4 +826,30 @@ case class Resolution( .get(dep.moduleVersion) .toSeq } yield (dep, err) + + def subset(dependencies: Set[Dependency]): Resolution = { + val (_, _, finalVersions) = nextDependenciesAndConflicts + + def updateVersion(dep: Dependency): Dependency = + dep.copy(version = finalVersions.getOrElse(dep.module, dep.version)) + + @tailrec def helper(current: Set[Dependency]): Set[Dependency] = { + val newDeps = current ++ current + .flatMap(finalDependencies0) + .map(updateVersion) + + val anyNewDep = (newDeps -- current).nonEmpty + + if (anyNewDep) + helper(newDeps) + else + newDeps + } + + copy( + rootDependencies = dependencies, + dependencies = helper(dependencies.map(updateVersion)) + // don't know if something should be done about conflicts + ) + } } diff --git a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala index 5e19adf25..640c0e499 100644 --- a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala +++ b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala @@ -7,7 +7,7 @@ import scala.annotation.tailrec sealed trait ResolutionProcess { def run[F[_]]( - fetch: ResolutionProcess.Fetch[F], + fetch: Fetch.Metadata[F], maxIterations: Int = -1 )(implicit F: Monad[F] @@ -34,7 +34,7 @@ sealed trait ResolutionProcess { } def next[F[_]]( - fetch: ResolutionProcess.Fetch[F] + fetch: Fetch.Metadata[F] )(implicit F: Monad[F] ): F[ResolutionProcess] = { @@ -58,7 +58,7 @@ case class Missing( cont: Resolution => ResolutionProcess ) extends ResolutionProcess { - def next(results: ResolutionProcess.FetchResult): ResolutionProcess = { + def next(results: Fetch.MD): ResolutionProcess = { val errors = results .collect{case (modVer, -\/(errs)) => modVer -> errs } @@ -72,7 +72,7 @@ case class Missing( val depMgmtMissing = depMgmtMissing0 -- results.map(_._1) def cont0(res: Resolution) = { - val res0 = + val res0 = successes.foldLeft(res){case (acc, (modVer, (source, proj))) => acc.copy(projectCache = acc.projectCache + ( modVer -> (source, acc.withDependencyManagement(proj)) @@ -121,12 +121,5 @@ object ResolutionProcess { else Missing(resolution0.missingFromCache.toSeq, resolution0, apply) } - - type FetchResult = Seq[( - (Module, String), - Seq[String] \/ (Artifact.Source, Project) - )] - - type Fetch[F[_]] = Seq[(Module, String)] => F[FetchResult] } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala new file mode 100644 index 000000000..fb14f9738 --- /dev/null +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -0,0 +1,272 @@ +package coursier.ivy + +import coursier.Fetch +import coursier.core._ +import scala.annotation.tailrec +import scala.util.matching.Regex +import scalaz._ +import java.util.regex.Pattern.quote + +object IvyRepository { + + val optionalPartRegex = (quote("(") + "[^" + quote("{()}") + "]*" + quote(")")).r + val variableRegex = (quote("[") + "[^" + quote("{[()]}") + "]*" + quote("]")).r + val propertyRegex = (quote("${") + "[^" + quote("{[()]}") + "]*" + quote("}")).r + + sealed abstract class PatternPart(val effectiveStart: Int, val effectiveEnd: Int) extends Product with Serializable { + require(effectiveStart <= effectiveEnd) + def start = effectiveStart + def end = effectiveEnd + + // FIXME Some kind of validation should be used here, to report all the missing variables, + // not only the first one missing. + def apply(content: String): Map[String, String] => String \/ String + } + object PatternPart { + case class Literal(override val effectiveStart: Int, override val effectiveEnd: Int) extends PatternPart(effectiveStart, effectiveEnd) { + def apply(content: String): Map[String, String] => String \/ String = { + assert(content.length == effectiveEnd - effectiveStart) + val matches = variableRegex.findAllMatchIn(content).toList + + variables => + @tailrec + def helper(idx: Int, matches: List[Regex.Match], b: StringBuilder): String \/ String = + if (idx >= content.length) + \/-(b.result()) + else { + assert(matches.headOption.forall(_.start >= idx)) + matches.headOption.filter(_.start == idx) match { + case Some(m) => + val variableName = content.substring(m.start + 1, m.end - 1) + variables.get(variableName) match { + case None => -\/(s"Variable not found: $variableName") + case Some(value) => + b ++= value + helper(m.end, matches.tail, b) + } + case None => + val nextIdx = matches.headOption.fold(content.length)(_.start) + b ++= content.substring(idx, nextIdx) + helper(nextIdx, matches, b) + } + } + + helper(0, matches, new StringBuilder) + } + } + case class Optional(start0: Int, end0: Int) extends PatternPart(start0 + 1, end0 - 1) { + override def start = start0 + override def end = end0 + + def apply(content: String): Map[String, String] => String \/ String = { + assert(content.length == effectiveEnd - effectiveStart) + val inner = Literal(effectiveStart, effectiveEnd).apply(content) + + variables => + \/-(inner(variables).fold(_ => "", x => x)) + } + } + } + + def substituteProperties(s: String, properties: Map[String, String]): String = + propertyRegex.findAllMatchIn(s).toVector.foldRight(s) { case (m, s0) => + val key = s0.substring(m.start + "${".length, m.end - "}".length) + val value = properties.getOrElse(key, "") + s0.take(m.start) + value + s0.drop(m.end) + } + +} + +case class IvyRepository( + pattern: String, + changing: Option[Boolean] = None, + properties: Map[String, String] = Map.empty, + withChecksums: Boolean = true, + withSignatures: Boolean = true, + withArtifacts: Boolean = true +) extends Repository { + + import Repository._ + import IvyRepository._ + + private val pattern0 = substituteProperties(pattern, properties) + + val parts = { + val optionalParts = optionalPartRegex.findAllMatchIn(pattern0).toList.map { m => + PatternPart.Optional(m.start, m.end) + } + + val len = pattern0.length + + @tailrec + def helper( + idx: Int, + opt: List[PatternPart.Optional], + acc: List[PatternPart] + ): Vector[PatternPart] = + if (idx >= len) + acc.toVector.reverse + else + opt match { + case Nil => + helper(len, Nil, PatternPart.Literal(idx, len) :: acc) + case (opt0 @ PatternPart.Optional(start0, end0)) :: rem => + if (idx < start0) + helper(start0, opt, PatternPart.Literal(idx, start0) :: acc) + else { + assert(idx == start0, s"idx: $idx, start0: $start0") + helper(end0, rem, opt0 :: acc) + } + } + + helper(0, optionalParts, Nil) + } + + assert(pattern0.isEmpty == parts.isEmpty) + if (pattern0.nonEmpty) { + for ((a, b) <- parts.zip(parts.tail)) + assert(a.end == b.start) + assert(parts.head.start == 0) + assert(parts.last.end == pattern0.length) + } + + private val substituteHelpers = parts.map { part => + part(pattern0.substring(part.effectiveStart, part.effectiveEnd)) + } + + def substitute(variables: Map[String, String]): String \/ String = + substituteHelpers.foldLeft[String \/ String](\/-("")) { + case (acc0, helper) => + for { + acc <- acc0 + s <- helper(variables) + } yield acc + s + } + + // See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a + // list of variables that should be supported. + // Some are missing (branch, conf, originalName). + private def variables( + module: Module, + version: String, + `type`: String, + artifact: String, + ext: String, + classifierOpt: Option[String] + ) = + Map( + "organization" -> module.organization, + "organisation" -> module.organization, + "orgPath" -> module.organization.replace('.', '/'), + "module" -> module.name, + "revision" -> version, + "type" -> `type`, + "artifact" -> artifact, + "ext" -> ext + ) ++ module.attributes ++ classifierOpt.map("classifier" -> _).toSeq + + + val source: Artifact.Source = + if (withArtifacts) + new Artifact.Source { + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ) = { + + val retained = + overrideClassifiers match { + case None => + project.publications.collect { + case (conf, p) + if conf == "*" || + conf == dependency.configuration || + project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) => + p + } + case Some(classifiers) => + val classifiersSet = classifiers.toSet + project.publications.collect { + case (_, p) if classifiersSet(p.classifier) => + p + } + } + + val retainedWithUrl = retained.flatMap { p => + substitute(variables( + dependency.module, + dependency.version, + p.`type`, + p.name, + p.ext, + Some(p.classifier).filter(_.nonEmpty) + )).toList.map(p -> _) + } + + retainedWithUrl.map { case (p, url) => + var artifact = Artifact( + url, + Map.empty, + Map.empty, + p.attributes, + changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable + ) + + if (withChecksums) + artifact = artifact.withDefaultChecksums + if (withSignatures) + artifact = artifact.withDefaultSignature + + artifact + } + } + } + else + Artifact.Source.empty + + + def find[F[_]]( + module: Module, + version: String, + fetch: Fetch.Content[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, (Artifact.Source, Project)] = { + + val eitherArtifact: String \/ Artifact = + for { + url <- substitute( + variables(module, version, "ivy", "ivy", "xml", None) + ) + } yield { + var artifact = Artifact( + url, + Map.empty, + Map.empty, + Attributes("ivy", ""), + changing = changing.getOrElse(version.contains("-SNAPSHOT")) + ) + + if (withChecksums) + artifact = artifact.withDefaultChecksums + if (withSignatures) + artifact = artifact.withDefaultSignature + + artifact + } + + for { + artifact <- EitherT(F.point(eitherArtifact)) + ivy <- fetch(artifact) + proj <- EitherT(F.point { + for { + xml <- \/.fromEither(compatibility.xmlParse(ivy)) + _ <- if (xml.label == "ivy-module") \/-(()) else -\/("Module definition not found") + proj <- IvyXml.project(xml) + } yield proj + }) + } yield (source, proj) + } + +} \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/ivy/IvyXml.scala b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala new file mode 100644 index 000000000..5f15e255d --- /dev/null +++ b/core/shared/src/main/scala/coursier/ivy/IvyXml.scala @@ -0,0 +1,172 @@ +package coursier.ivy + +import coursier.core._ +import coursier.util.Xml._ + +import scalaz.{ Node => _, _ }, Scalaz._ + +object IvyXml { + + val attributesNamespace = "http://ant.apache.org/ivy/extra" + + private def info(node: Node): String \/ (Module, String) = + for { + org <- node.attribute("organisation") + name <- node.attribute("module") + version <- node.attribute("revision") + attr = node.attributesFromNamespace(attributesNamespace) + } yield (Module(org, name, attr.toMap), version) + + // FIXME Errors are ignored here + private def configurations(node: Node): Seq[(String, Seq[String])] = + node.children + .filter(_.label == "conf") + .flatMap { node => + node.attribute("name").toOption.toSeq.map(_ -> node) + } + .map { case (name, node) => + name -> node.attribute("extends").toOption.toSeq.flatMap(_.split(',')) + } + + // FIXME "default(compile)" likely not to be always the default + def mappings(mapping: String): Seq[(String, String)] = + mapping.split(';').flatMap { m => + val (froms, tos) = m.split("->", 2) match { + case Array(from) => (from, "default(compile)") + case Array(from, to) => (from, to) + } + + for { + from <- froms.split(',') + to <- tos.split(',') + } yield (from.trim, to.trim) + } + + // FIXME Errors ignored as above - warnings should be reported at least for anything suspicious + private def dependencies(node: Node): Seq[(String, Dependency)] = + node.children + .filter(_.label == "dependency") + .flatMap { node => + // artifact and include sub-nodes are ignored here + + val excludes = node.children + .filter(_.label == "exclude") + .flatMap { node0 => + val org = node.attribute("org").getOrElse("*") + val name = node.attribute("module").orElse(node.attribute("name")).getOrElse("*") + val confs = node.attribute("conf").toOption.fold(Seq("*"))(_.split(',')) + confs.map(_ -> (org, name)) + } + .groupBy { case (conf, _) => conf } + .map { case (conf, l) => conf -> l.map { case (_, e) => e }.toSet } + + val allConfsExcludes = excludes.getOrElse("*", Set.empty) + + for { + org <- node.attribute("org").toOption.toSeq + name <- node.attribute("name").toOption.toSeq + version <- node.attribute("rev").toOption.toSeq + rawConf <- node.attribute("conf").toOption.toSeq + (fromConf, toConf) <- mappings(rawConf) + attr = node.attributesFromNamespace(attributesNamespace) + } yield fromConf -> Dependency( + Module(org, name, attr.toMap), + version, + toConf, + allConfsExcludes ++ excludes.getOrElse(fromConf, Set.empty), + Attributes("jar", ""), // should come from possible artifact nodes + optional = false + ) + } + + private def publications(node: Node): Map[String, Seq[Publication]] = + node.children + .filter(_.label == "artifact") + .flatMap { node => + val name = node.attribute("name").getOrElse("") + val type0 = node.attribute("type").getOrElse("jar") + val ext = node.attribute("ext").getOrElse(type0) + val confs = node.attribute("conf").toOption.fold(Seq("*"))(_.split(',')) + val classifier = node.attribute("classifier").toOption.getOrElse("") + confs.map(_ -> Publication(name, type0, ext, classifier)) + } + .groupBy { case (conf, _) => conf } + .map { case (conf, l) => conf -> l.map { case (_, p) => p } } + + def project(node: Node): String \/ Project = + for { + infoNode <- node.children + .find(_.label == "info") + .toRightDisjunction("Info not found") + + (module, version) <- info(infoNode) + + dependenciesNodeOpt = node.children + .find(_.label == "dependencies") + + dependencies0 = dependenciesNodeOpt.map(dependencies).getOrElse(Nil) + + configurationsNodeOpt = node.children + .find(_.label == "configurations") + + configurationsOpt = configurationsNodeOpt.map(configurations) + + configurations0 = configurationsOpt.getOrElse(Seq("default" -> Seq.empty[String])) + + publicationsNodeOpt = node.children + .find(_.label == "publications") + + publicationsOpt = publicationsNodeOpt.map(publications) + + } yield { + + val description = infoNode.children + .find(_.label == "description") + .map(_.textContent.trim) + .getOrElse("") + + val licenses = infoNode.children + .filter(_.label == "license") + .flatMap { n => + n.attribute("name").toOption.map { name => + (name, n.attribute("url").toOption) + }.toSeq + } + + val publicationDate = infoNode.attribute("publication") + .toOption + .flatMap(parseDateTime) + + Project( + module, + version, + dependencies0, + configurations0.toMap, + None, + Nil, + Nil, + Nil, + None, + None, + if (publicationsOpt.isEmpty) + // no publications node -> default JAR artifact + Seq("*" -> Publication(module.name, "jar", "jar", "")) + else { + // publications node is there -> only its content (if it is empty, no artifacts, + // as per the Ivy manual) + val inAllConfs = publicationsOpt.flatMap(_.get("*")).getOrElse(Nil) + configurations0.flatMap { case (conf, _) => + (publicationsOpt.flatMap(_.get(conf)).getOrElse(Nil) ++ inAllConfs).map(conf -> _) + } + }, + Info( + description, + "", + licenses, + Nil, + publicationDate + ) + ) + } + +} \ No newline at end of file diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 3ea96fc81..821cd1866 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -1,5 +1,6 @@ package coursier.maven +import coursier.Fetch import coursier.core._ import coursier.core.compatibility.encodeURIComponent @@ -10,6 +11,7 @@ object MavenRepository { def ivyLikePath( org: String, + dirName: String, name: String, version: String, subDir: String, @@ -18,7 +20,7 @@ object MavenRepository { ) = Seq( org, - name, + dirName, version, subDir, s"$name$baseSuffix.$ext" @@ -35,18 +37,40 @@ object MavenRepository { .map(_.value) .filter(_.nonEmpty) + + val defaultConfigurations = Map( + "compile" -> Seq.empty, + "runtime" -> Seq("compile"), + "default" -> Seq("runtime"), + "test" -> Seq("runtime") + ) + + def dirModuleName(module: Module, sbtAttrStub: Boolean): String = + if (sbtAttrStub) { + var name = module.name + for (scalaVersion <- module.attributes.get("scalaVersion")) + name = name + "_" + scalaVersion + for (sbtVersion <- module.attributes.get("sbtVersion")) + name = name + "_" + sbtVersion + name + } else + module.name + } case class MavenRepository( root: String, - ivyLike: Boolean = false + ivyLike: Boolean = false, + changing: Option[Boolean] = None, + /** Hackish hack for sbt plugins mainly - what this does really sucks */ + sbtAttrStub: Boolean = false ) extends Repository { import Repository._ import MavenRepository._ val root0 = if (root.endsWith("/")) root else root + "/" - val source = MavenSource(root0, ivyLike) + val source = MavenSource(root0, ivyLike, changing, sbtAttrStub) def projectArtifact( module: Module, @@ -58,6 +82,7 @@ case class MavenRepository( if (ivyLike) ivyLikePath( module.organization, + dirModuleName(module, sbtAttrStub), // maybe not what we should do here, don't know module.name, versioningValue getOrElse version, "poms", @@ -66,7 +91,7 @@ case class MavenRepository( ) else module.organization.split('.').toSeq ++ Seq( - module.name, + dirModuleName(module, sbtAttrStub), version, s"${module.name}-${versioningValue getOrElse version}.pom" ) @@ -76,7 +101,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = changing.getOrElse(version.contains("-SNAPSHOT")) ) .withDefaultChecksums .withDefaultSignature @@ -87,7 +113,7 @@ case class MavenRepository( else { val path = ( module.organization.split('.').toSeq ++ Seq( - module.name, + dirModuleName(module, sbtAttrStub), "maven-metadata.xml" ) ) .map(encodeURIComponent) @@ -97,10 +123,11 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = true ) .withDefaultChecksums - .withDefaultChecksums + .withDefaultSignature Some(artifact) } @@ -113,7 +140,7 @@ case class MavenRepository( else { val path = ( module.organization.split('.').toSeq ++ Seq( - module.name, + dirModuleName(module, sbtAttrStub), version, "maven-metadata.xml" ) @@ -124,7 +151,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = true ) .withDefaultChecksums .withDefaultSignature @@ -134,7 +162,7 @@ case class MavenRepository( def versions[F[_]]( module: Module, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, Versions] = @@ -156,7 +184,7 @@ case class MavenRepository( def snapshotVersioning[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, SnapshotVersioning] = { @@ -180,7 +208,7 @@ case class MavenRepository( def findNoInterval[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, Project] = @@ -219,7 +247,7 @@ case class MavenRepository( module: Module, version: String, versioningValue: Option[String], - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, Project] = { @@ -231,7 +259,14 @@ case class MavenRepository( xml <- \/.fromEither(compatibility.xmlParse(str)) _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") proj <- Pom.project(xml) - } yield proj): (String \/ Project) + } yield proj.copy( + configurations = defaultConfigurations, + publications = Seq( + "compile" -> Publication(module.name, "jar", "jar", ""), + "docs" -> Publication(module.name, "doc", "jar", "javadoc"), + "sources" -> Publication(module.name, "src", "jar", "sources") + ) + )): (String \/ Project) } } } @@ -240,7 +275,7 @@ case class MavenRepository( def find[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ): EitherT[F, String, (Artifact.Source, Project)] = { diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 02c780d3f..f07a4f5b3 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -2,78 +2,114 @@ package coursier.maven import coursier.core._ -case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { +case class MavenSource( + root: String, + ivyLike: Boolean, + changing: Option[Boolean] = None, + /** See doc on MavenRepository */ + sbtAttrStub: Boolean +) extends Artifact.Source { + import Repository._ import MavenRepository._ + private implicit class DocSourcesArtifactExtensions(val underlying: Artifact) { + def withJavadocSources: Artifact = { + val base = underlying.url.stripSuffix(".jar") + underlying.copy(extra = underlying.extra ++ Seq( + "sources" -> Artifact( + base + "-sources.jar", + Map.empty, + Map.empty, + Attributes("jar", "src"), // Are these the right attributes? + changing = underlying.changing + ) + .withDefaultChecksums + .withDefaultSignature, + "javadoc" -> Artifact( + base + "-javadoc.jar", + Map.empty, + Map.empty, + Attributes("jar", "javadoc"), // Same comment as above + changing = underlying.changing + ) + .withDefaultChecksums + .withDefaultSignature + )) + } + } + def artifacts( dependency: Dependency, - project: Project + project: Project, + overrideClassifiers: Option[Seq[String]] ): Seq[Artifact] = { - def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = - ivyLikePath( - dependency.module.organization, - dependency.module.name, - project.version, - subDir, - baseSuffix, - ext - ) - - val path = - if (ivyLike) - ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) - else { - val versioning = - project - .snapshotVersioning - .flatMap(versioning => - mavenVersioning(versioning, dependency.attributes.classifier, dependency.attributes.`type`) - ) - - dependency.module.organization.split('.').toSeq ++ Seq( - dependency.module.name, + def artifactOf(module: Module, publication: Publication) = { + def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = + ivyLikePath( + module.organization, + MavenRepository.dirModuleName(module, sbtAttrStub), + module.name, project.version, - s"${dependency.module.name}-${versioning getOrElse project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + subDir, + baseSuffix, + ext ) + + val path = + if (ivyLike) + ivyLikePath0(publication.`type` + "s", "", publication.ext) + else { + val versioning = + project + .snapshotVersioning + .flatMap(versioning => + mavenVersioning(versioning, publication.classifier, publication.`type`) + ) + + module.organization.split('.').toSeq ++ Seq( + MavenRepository.dirModuleName(module, sbtAttrStub), + project.version, + s"${module.name}-${versioning getOrElse project.version}${Some(publication.classifier).filter(_.nonEmpty).map("-" + _).mkString}.${publication.ext}" + ) + } + + val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT")) + var artifact = + Artifact( + root + path.mkString("/"), + Map.empty, + Map.empty, + publication.attributes, + changing = changing0 + ) + .withDefaultChecksums + + if (publication.ext == "jar") { + artifact = artifact.withDefaultSignature } - var artifact = - Artifact( - root + path.mkString("/"), - Map.empty, - Map.empty, - dependency.attributes - ) - .withDefaultChecksums - - if (dependency.attributes.`type` == "jar") { - artifact = artifact.withDefaultSignature - - // FIXME Snapshot versioning of sources and javadoc is not taken into account here. - // Will be ok if it's the same as the main JAR though. - - artifact = - if (ivyLike) { - val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") - val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") - - artifact - .copy( - extra = artifact.extra ++ Map( - "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? - .withDefaultChecksums - .withDefaultSignature, - "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above - .withDefaultChecksums - .withDefaultSignature - )) - } else - artifact - .withJavadocSources + artifact } - Seq(artifact) + overrideClassifiers match { + case Some(classifiers) => + val classifiersSet = classifiers.toSet + project.publications.collect { + case (_, p) if classifiersSet(p.classifier) => + artifactOf(dependency.module, p) + } + case None => + Seq( + artifactOf( + dependency.module, + dependency.attributes.publication( + dependency.module.name, + dependency.attributes.`type` + ) + ) + ) + } } } diff --git a/core/shared/src/main/scala/coursier/maven/Pom.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala index 8033498fa..7f42eeaab 100644 --- a/core/shared/src/main/scala/coursier/maven/Pom.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -7,21 +7,6 @@ import scalaz._ object Pom { import coursier.util.Xml._ - object Text { - def unapply(n: Node): Option[String] = - if (n.isText) Some(n.textContent) - else None - } - - private def text(elem: Node, label: String, description: String) = { - import Scalaz.ToOptionOpsFromOption - - elem.child - .find(_.label == label) - .flatMap(_.child.collectFirst{case Text(t) => t}) - .toRightDisjunction(s"$description not found") - } - def property(elem: Node): String \/ (String, String) = { // Not matching with Text, which fails on scala-js if the property value has xml comments if (elem.isElement) \/-(elem.label -> elem.textContent) @@ -37,39 +22,37 @@ object Pom { else e } name <- text(node, "artifactId", "Name") - } yield Module(organization, name).trim + } yield Module(organization, name, Map.empty).trim } private def readVersion(node: Node) = text(node, "version", "Version").getOrElse("").trim - private val defaultScope = Scope.Other("") private val defaultType = "jar" private val defaultClassifier = "" - def dependency(node: Node): String \/ Dependency = { + def dependency(node: Node): String \/ (String, Dependency) = { for { mod <- module(node) version0 = readVersion(node) scopeOpt = text(node, "scope", "").toOption - .map(Parse.scope) typeOpt = text(node, "type", "").toOption classifierOpt = text(node, "classifier", "").toOption - xmlExclusions = node.child + xmlExclusions = node.children .find(_.label == "exclusions") - .map(_.child.filter(_.label == "exclusion")) + .map(_.children.filter(_.label == "exclusion")) .getOrElse(Seq.empty) exclusions <- { import Scalaz._ xmlExclusions.toList.traverseU(module(_)) } optional = text(node, "optional", "").toOption.toSeq.contains("true") - } yield Dependency( + } yield scopeOpt.getOrElse("") -> Dependency( mod, version0, - scopeOpt getOrElse defaultScope, - Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), + "", exclusions.map(mod => (mod.organization, mod.name)).toSet, + Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier), optional ) } @@ -82,7 +65,7 @@ object Pom { case _ => None } - val properties = node.child + val properties = node.children .filter(_.label == "property") .flatMap{ p => for{ @@ -99,28 +82,28 @@ object Pom { val id = text(node, "id", "Profile ID").getOrElse("") - val xmlActivationOpt = node.child + val xmlActivationOpt = node.children .find(_.label == "activation") val (activeByDefault, activation) = xmlActivationOpt.fold((Option.empty[Boolean], Activation(Nil)))(profileActivation) - val xmlDeps = node.child + val xmlDeps = node.children .find(_.label == "dependencies") - .map(_.child.filter(_.label == "dependency")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) for { deps <- xmlDeps.toList.traverseU(dependency) - xmlDepMgmts = node.child + xmlDepMgmts = node.children .find(_.label == "dependencyManagement") - .flatMap(_.child.find(_.label == "dependencies")) - .map(_.child.filter(_.label == "dependency")) + .flatMap(_.children.find(_.label == "dependencies")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) depMgmts <- xmlDepMgmts.toList.traverseU(dependency) - xmlProperties = node.child + xmlProperties = node.children .find(_.label == "properties") - .map(_.child.collect{case elem if elem.isElement => elem}) + .map(_.children.collect{case elem if elem.isElement => elem}) .getOrElse(Seq.empty) properties <- { @@ -138,7 +121,7 @@ object Pom { projModule <- module(pom, groupIdIsOptional = true) projVersion = readVersion(pom) - parentOpt = pom.child + parentOpt = pom.children .find(_.label == "parent") parentModuleOpt <- parentOpt .map(module(_).map(Some(_))) @@ -146,16 +129,16 @@ object Pom { parentVersionOpt = parentOpt .map(readVersion) - xmlDeps = pom.child + xmlDeps = pom.children .find(_.label == "dependencies") - .map(_.child.filter(_.label == "dependency")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) deps <- xmlDeps.toList.traverseU(dependency) - xmlDepMgmts = pom.child + xmlDepMgmts = pom.children .find(_.label == "dependencyManagement") - .flatMap(_.child.find(_.label == "dependencies")) - .map(_.child.filter(_.label == "dependency")) + .flatMap(_.children.find(_.label == "dependencies")) + .map(_.children.filter(_.label == "dependency")) .getOrElse(Seq.empty) depMgmts <- xmlDepMgmts.toList.traverseU(dependency) @@ -173,43 +156,94 @@ object Pom { .map(mod => if (mod.organization.isEmpty) -\/("Parent organization missing") else \/-(())) .getOrElse(\/-(())) - xmlProperties = pom.child + xmlProperties = pom.children .find(_.label == "properties") - .map(_.child.collect{case elem if elem.isElement => elem}) + .map(_.children.collect{case elem if elem.isElement => elem}) .getOrElse(Seq.empty) properties <- xmlProperties.toList.traverseU(property) - xmlProfiles = pom.child + xmlProfiles = pom.children .find(_.label == "profiles") - .map(_.child.filter(_.label == "profile")) + .map(_.children.filter(_.label == "profile")) .getOrElse(Seq.empty) profiles <- xmlProfiles.toList.traverseU(profile) - } yield Project( - projModule.copy(organization = groupId), - version, - deps, - parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), - depMgmts, - properties.toMap, - profiles, - None, - None - ) - } + extraAttrs <- properties + .collectFirst { case ("extraDependencyAttributes", s) => extraAttributes(s) } + .getOrElse(\/-(Map.empty)) - def parseDateTime(s: String): Option[Versions.DateTime] = - if (s.length == 14 && s.forall(_.isDigit)) - Some(Versions.DateTime( - s.substring(0, 4).toInt, - s.substring(4, 6).toInt, - s.substring(6, 8).toInt, - s.substring(8, 10).toInt, - s.substring(10, 12).toInt, - s.substring(12, 14).toInt - )) - else - None + extraAttrsMap = extraAttrs.map { + case (mod, ver) => + (mod.copy(attributes = Map.empty), ver) -> mod.attributes + }.toMap + + } yield { + + val description = pom.children + .find(_.label == "description") + .map(_.textContent) + .getOrElse("") + + val homePage = pom.children + .find(_.label == "url") + .map(_.textContent) + .getOrElse("") + + val licenses = pom.children + .find(_.label == "licenses") + .toSeq + .flatMap(_.children) + .filter(_.label == "license") + .flatMap { n => + text(n, "name", "License name").toOption.map { name => + (name, text(n, "url", "License URL").toOption) + }.toSeq + } + + val developers = pom.children + .find(_.label == "developers") + .toSeq + .flatMap(_.children) + .filter(_.label == "developer") + .map { n => + for { + id <- text(n, "id", "Developer ID") + name <- text(n, "name", "Developer name") + url <- text(n, "url", "Developer URL") + } yield Info.Developer(id, name, url) + } + .collect { + case \/-(d) => d + } + + Project( + projModule.copy(organization = groupId), + version, + deps.map { + case (config, dep0) => + val dep = extraAttrsMap.get(dep0.moduleVersion).fold(dep0)(attrs => + dep0.copy(module = dep0.module.copy(attributes = attrs)) + ) + config -> dep + }, + Map.empty, + parentModuleOpt.map((_, parentVersionOpt.getOrElse(""))), + depMgmts, + properties, + profiles, + None, + None, + Nil, + Info( + description, + homePage, + licenses, + developers, + None + ) + ) + } + } def versions(node: Node): String \/ Versions = { import Scalaz._ @@ -218,7 +252,7 @@ object Pom { organization <- text(node, "groupId", "Organization") // Ignored name <- text(node, "artifactId", "Name") // Ignored - xmlVersioning <- node.child + xmlVersioning <- node.children .find(_.label == "versioning") .toRightDisjunction("Versioning info not found in metadata") @@ -227,9 +261,9 @@ object Pom { release = text(xmlVersioning, "release", "Release version") .getOrElse("") - versionsOpt = xmlVersioning.child + versionsOpt = xmlVersioning.children .find(_.label == "versions") - .map(_.child.filter(_.label == "version").flatMap(_.child.collectFirst{case Text(t) => t})) + .map(_.children.filter(_.label == "version").flatMap(_.children.collectFirst{case Text(t) => t})) lastUpdatedOpt = text(xmlVersioning, "lastUpdated", "Last update date and time") .toOption @@ -269,7 +303,7 @@ object Pom { name <- text(node, "artifactId", "Name") version = readVersion(node) - xmlVersioning <- node.child + xmlVersioning <- node.children .find(_.label == "versioning") .toRightDisjunction("Versioning info not found in metadata") @@ -278,15 +312,15 @@ object Pom { release = text(xmlVersioning, "release", "Release version") .getOrElse("") - versionsOpt = xmlVersioning.child + versionsOpt = xmlVersioning.children .find(_.label == "versions") - .map(_.child.filter(_.label == "version").flatMap(_.child.collectFirst{case Text(t) => t})) + .map(_.children.filter(_.label == "version").flatMap(_.children.collectFirst{case Text(t) => t})) lastUpdatedOpt = text(xmlVersioning, "lastUpdated", "Last update date and time") .toOption .flatMap(parseDateTime) - xmlSnapshotOpt = xmlVersioning.child + xmlSnapshotOpt = xmlVersioning.children .find(_.label == "snapshot") timestamp = xmlSnapshotOpt @@ -314,15 +348,15 @@ object Pom { case "false" => false } - xmlSnapshotVersions = xmlVersioning.child + xmlSnapshotVersions = xmlVersioning.children .find(_.label == "snapshotVersions") - .map(_.child.filter(_.label == "snapshotVersion")) + .map(_.children.filter(_.label == "snapshotVersion")) .getOrElse(Seq.empty) snapshotVersions <- xmlSnapshotVersions .toList .traverseU(snapshotVersion) } yield SnapshotVersioning( - Module(organization, name), + Module(organization, name, Map.empty), version, latest, release, @@ -333,4 +367,69 @@ object Pom { snapshotVersions ) } + + val extraAttributeSeparator = ":#@#:" + val extraAttributePrefix = "+" + + val extraAttributeOrg = "organisation" + val extraAttributeName = "module" + val extraAttributeVersion = "revision" + + val extraAttributeBase = Set( + extraAttributeOrg, + extraAttributeName, + extraAttributeVersion, + "branch" + ) + + val extraAttributeDropPrefix = "e:" + + def extraAttribute(s: String): String \/ (Module, String) = { + // vaguely does the same as: + // https://github.com/apache/ant-ivy/blob/2.2.0/src/java/org/apache/ivy/core/module/id/ModuleRevisionId.java#L291 + + // dropping the attributes with a value of NULL here... + + val rawParts = s.split(extraAttributeSeparator).toSeq + + val partsOrError = + if (rawParts.length % 2 == 0) { + val malformed = rawParts.filter(!_.startsWith(extraAttributePrefix)) + if (malformed.isEmpty) + \/-(rawParts.map(_.drop(extraAttributePrefix.length))) + else + -\/(s"Malformed attributes ${malformed.map("'"+_+"'").mkString(", ")} in extra attributes '$s'") + } else + -\/(s"Malformed extra attributes '$s'") + + def attrFrom(attrs: Map[String, String], name: String): String \/ String = + \/.fromEither( + attrs.get(name) + .toRight(s"$name not found in extra attributes '$s'") + ) + + for { + parts <- partsOrError + attrs = parts.grouped(2).collect { + case Seq(k, v) if v != "NULL" => + k.stripPrefix(extraAttributeDropPrefix) -> v + }.toMap + org <- attrFrom(attrs, extraAttributeOrg) + name <- attrFrom(attrs, extraAttributeName) + version <- attrFrom(attrs, extraAttributeVersion) + remainingAttrs = attrs.filterKeys(!extraAttributeBase(_)) + } yield (Module(org, name, remainingAttrs.toVector.toMap), version) + } + + def extraAttributes(s: String): String \/ Seq[(Module, String)] = { + val lines = s.split('\n').toSeq.map(_.trim).filter(_.nonEmpty) + + lines.foldLeft[String \/ Seq[(Module, String)]](\/-(Vector.empty)) { + case (acc, line) => + for { + modVers <- acc + modVer <- extraAttribute(line) + } yield modVers :+ modVer + } + } } diff --git a/core/shared/src/main/scala/coursier/package.scala b/core/shared/src/main/scala/coursier/package.scala index 6803f43ed..8a19dd3cc 100644 --- a/core/shared/src/main/scala/coursier/package.scala +++ b/core/shared/src/main/scala/coursier/package.scala @@ -9,8 +9,8 @@ package object coursier { def apply( module: Module, version: String, - // Substituted by Resolver with its own default scope (compile) - scope: Scope = Scope.Other(""), + // Substituted by Resolver with its own default configuration (compile) + configuration: String = "", attributes: Attributes = Attributes(), exclusions: Set[(String, String)] = Set.empty, optional: Boolean = false @@ -18,9 +18,9 @@ package object coursier { core.Dependency( module, version, - scope, - attributes, + configuration, exclusions, + attributes, optional ) } @@ -37,19 +37,20 @@ package object coursier { type Project = core.Project val Project = core.Project + type Info = core.Info + val Info = core.Info + type Profile = core.Profile val Profile = core.Profile type Module = core.Module object Module { - def apply(organization: String, name: String): Module = - core.Module(organization, name) + def apply(organization: String, name: String, attributes: Map[String, String] = Map.empty): Module = + core.Module(organization, name, attributes) } type ModuleVersion = (core.Module, String) - type Scope = core.Scope - val Scope = core.Scope type Repository = core.Repository val Repository = core.Repository diff --git a/core/shared/src/main/scala/coursier/util/Xml.scala b/core/shared/src/main/scala/coursier/util/Xml.scala index 5518a58d2..a7469d659 100644 --- a/core/shared/src/main/scala/coursier/util/Xml.scala +++ b/core/shared/src/main/scala/coursier/util/Xml.scala @@ -1,14 +1,33 @@ package coursier.util +import coursier.core.Versions + +import scalaz.{\/-, -\/, \/, Scalaz} + object Xml { /** A representation of an XML node/document, with different implementations on JVM and JS */ trait Node { def label: String - def child: Seq[Node] + /** Namespace / key / value */ + def attributes: Seq[(String, String, String)] + def children: Seq[Node] def isText: Boolean def textContent: String def isElement: Boolean + + def attributesFromNamespace(namespace: String): Seq[(String, String)] = + attributes.collect { + case (`namespace`, k, v) => + k -> v + } + + lazy val attributesMap = attributes.map { case (_, k, v) => k -> v }.toMap + def attribute(name: String): String \/ String = + attributesMap.get(name) match { + case None => -\/(s"Missing attribute $name") + case Some(value) => \/-(value) + } } object Node { @@ -16,10 +35,39 @@ object Xml { new Node { val isText = false val isElement = false - val child = Nil + val children = Nil val label = "" + val attributes = Nil val textContent = "" } } + object Text { + def unapply(n: Node): Option[String] = + if (n.isText) Some(n.textContent) + else None + } + + def text(elem: Node, label: String, description: String) = { + import Scalaz.ToOptionOpsFromOption + + elem.children + .find(_.label == label) + .flatMap(_.children.collectFirst{case Text(t) => t}) + .toRightDisjunction(s"$description not found") + } + + def parseDateTime(s: String): Option[Versions.DateTime] = + if (s.length == 14 && s.forall(_.isDigit)) + Some(Versions.DateTime( + s.substring(0, 4).toInt, + s.substring(4, 6).toInt, + s.substring(6, 8).toInt, + s.substring(8, 10).toInt, + s.substring(10, 12).toInt, + s.substring(12, 14).toInt + )) + else + None + } diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 000000000..16c749903 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,187 @@ +# Coursier + +*Pure Scala Artifact Fetching* + +A pure Scala substitute for [Aether](http://www.eclipse.org/aether/) + +[![Build Status](https://travis-ci.org/alexarchambault/coursier.svg?branch=master)](https://travis-ci.org/alexarchambault/coursier) +[![Build status (Windows)](https://ci.appveyor.com/api/projects/status/trtum5b7washfbj9?svg=true)](https://ci.appveyor.com/project/alexarchambault/coursier) +[![Join the chat at https://gitter.im/alexarchambault/coursier](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/alexarchambault/coursier?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Maven Central](https://img.shields.io/maven-central/v/com.github.alexarchambault/coursier_2.11.svg)](https://maven-badges.herokuapp.com/maven-central/com.github.alexarchambault/coursier_2.11) + +*coursier* is a dependency resolver / fetcher *à la* Maven / Ivy, entirely +rewritten from scratch in Scala. It aims at being fast and easy to embed +in other contexts. Its very core (`core` module) aims at being +extremely pure, and should be approached thinking algebraically. + +The `files` module handles caching of the metadata and artifacts themselves, +and is less so pure than the `core` module, in the sense that it happily +does IO as a side-effect (although it naturally favors immutability for all +that's kept in memory). + +It handles fancy Maven features like +* [POM inheritance](http://books.sonatype.com/mvnref-book/reference/pom-relationships-sect-project-relationships.html#pom-relationships-sect-project-inheritance), +* [dependency management](http://books.sonatype.com/mvnex-book/reference/optimizing-sect-dependencies.html), +* [import scope](https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Importing_Dependencies), +* [properties](http://books.sonatype.com/mvnref-book/reference/resource-filtering-sect-properties.html), +* etc. + +It happily resolves dependencies involving modules from the Hadoop ecosystem (Spark, Flink, etc.), that +make a heavy use of these. + +It can be used either from the command-line, via its API, or from the browser. + +It downloads the metadata or the artifacts in parallel (usually, 6 parallel +downloads). + +## Command-line + +Download and run its laucher with +``` +$ curl -L -o coursier https://git.io/vBSmI && chmod +x coursier && ./coursier --help +``` + +Note that the launcher itself weights only 8 kB and can be easily +embedded as is in other projects. +The first time it is run, it will download the artifacts required to launch +coursier. You'll be fine the next times :-). + +The cache of this default launcher defaults to a directory named `.coursier`, +in the same directory as the launcher. This can be changed by manually adjusting +the `COURSIER_CACHE` variable in the first lines of the launcher. + +``` +$ ./coursier --help +``` +lists the available coursier commands. The most notable ones are `launch`, +`fetch`, and `classpath`. Type +``` +$ ./coursier command --help +``` +to get a description of the various options the command `command` (replace with one +of the above command) accepts. + +### launch + +The `launch` command fetches a set of Maven coordinates it is given, along +with their transitive dependencies, then launches the "main `main` class" from +it if it can find one (typically from the manifest of the first coordinates). +The main class to launch can also be manually specified with the `-M` option. + +For example, it can launch: + +* [Ammonite](https://github.com/lihaoyi/Ammonite) (enhanced Scala REPL), +``` +$ ./coursier launch com.lihaoyi:ammonite-repl_2.11.7:0.5.0 +``` + +along with the REPLs of various JVM languages like + +* Frege, +``` +$ ./coursier launch -r central -r https://oss.sonatype.org/content/groups/public \ + org.frege-lang:frege-repl-core:1.3 -M frege.repl.FregeRepl +``` + +* clojure, +``` +$ ./coursier launch org.clojure:clojure:1.7.0 -M clojure.main +``` + +* jruby, +``` +$ wget https://raw.githubusercontent.com/jruby/jruby/master/bin/jirb && \ + ./coursier launch org.jruby:jruby:9.0.4.0 -M org.jruby.Main -- -- jirb +``` + +* jython, +``` +$ ./coursier launch org.python:jython-standalone:2.7.0 -M org.python.util.jython +``` + +* Groovy, +``` +$ ./coursier launch org.codehaus.groovy:groovy-groovysh:2.4.5 -M org.codehaus.groovy.tools.shell.Main \ + commons-cli:commons-cli:1.3.1 +``` + +etc. + +and various programs, like + +* Proguard and its utility Retrace, +``` +$ ./coursier launch net.sf.proguard:proguard-base:5.2.1 -M proguard.ProGuard +$ ./coursier launch net.sf.proguard:proguard-retrace:5.2.1 -M proguard.retrace.ReTrace +``` + +### fetch + +The `fetch` command simply fetches a set of dependencies, along with their +transitive dependencies, then prints the local paths of all their artefacts. + +Example +``` +$ ./coursier fetch org.apache.spark:spark-sql_2.11:1.5.2 +... +/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/io/dropwizard/metrics/metrics-jvm/3.1.2/metrics-jvm-3.1.2.jar +/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/javax/servlet/javax.servlet-api/3.0.1/javax.servlet-api-3.0.1.jar +/path/to/.coursier/cache/0.1.0-SNAPSHOT-2f5e731/files/central/javax/inject/javax.inject/1/javax.inject-1.jar +... +``` + +### classpath + +The `classpath` command transitively fetches a set of dependencies like +`fetch` does, then prints a classpath that can be handed over directly +to `java`, like +``` +$ java -cp "$(./coursier classpath com.lihaoyi:ammonite-repl_2.11.7:0.5.0 | tail -n1)" ammonite.repl.Repl +Loading... +Welcome to the Ammonite Repl 0.5.0 +(Scala 2.11.7 Java 1.8.0_60) +@ +``` + +## API + +This [gist](https://gist.github.com/larsrh/42da43aa74dc4e78aa59) by [Lars Hupel](https://github.com/larsrh/) +illustrates how the API of coursier can be used to get transitives dependencies +and fetch the corresponding artefacts. + +More explanations to come :-) + +## Scala JS demo + +*coursier* is also compiled to Scala JS, and can be tested in the browser via its +[demo](http://alexarchambault.github.io/coursier/#demo). + +# To do / missing + +- Snapshots metadata / artifacts, once in cache, are not automatically +updated for now. [#41](https://github.com/alexarchambault/coursier/issues/41) +- File locking could be better (none for metadata, no re-attempt if file locked elsewhere for artifacts) [#71](https://github.com/alexarchambault/coursier/issues/71) +- Handle "configurations" like Ivy does, instead of just the standard +(hard-coded) Maven "scopes" [#8](https://github.com/alexarchambault/coursier/issues/8) +- SBT plugin [#52](https://github.com/alexarchambault/coursier/issues/52), +requires Ivy-like configurations [#8](https://github.com/alexarchambault/coursier/issues/8) + +See the list of [issues](https://github.com/alexarchambault/coursier/issues). + +# Contributors + +- Your name here :-) + +Don't hesitate to pick an issue to contribute, and / or ask for help for how to proceed +on the [Gitter channel](https://gitter.im/alexarchambault/coursier). + +# Projects using coursier + +- [Lars Hupel](https://github.com/larsrh/)'s [libisabelle](https://github.com/larsrh/libisabelle) fetches +some of its requirements via coursier, +- [jupyter-scala](https://github.com/alexarchambault/jupyter-scala) should soon allow +to add dependencies in its sessions with coursier (initial motivation for writing coursier), +- Your project here :-) + + +Released under the Apache license, v2. diff --git a/fetch-js/src/main/scala/coursier/Fetch.scala b/fetch-js/src/main/scala/coursier/Fetch.scala deleted file mode 100644 index 3e7d220d5..000000000 --- a/fetch-js/src/main/scala/coursier/Fetch.scala +++ /dev/null @@ -1,26 +0,0 @@ -package coursier - -import scalaz.concurrent.Task - -object Fetch { - - implicit def default( - repositories: Seq[core.Repository] - ): ResolutionProcess.Fetch[Task] = - apply(repositories, Platform.artifact) - - def apply( - repositories: Seq[core.Repository], - fetch: Repository.Fetch[Task] - ): ResolutionProcess.Fetch[Task] = { - - modVers => Task.gatherUnordered( - modVers.map { case (module, version) => - Repository.find(repositories, module, version, fetch) - .run - .map((module, version) -> _) - } - ) - } - -} diff --git a/fetch-js/src/main/scala/coursier/Platform.scala b/fetch-js/src/main/scala/coursier/Platform.scala index 799c20fb5..a9e412e33 100644 --- a/fetch-js/src/main/scala/coursier/Platform.scala +++ b/fetch-js/src/main/scala/coursier/Platform.scala @@ -8,7 +8,6 @@ import scala.scalajs.js import js.Dynamic.{ global => g } import scala.scalajs.js.timers._ -import scalaz.concurrent.Task import scalaz.{ -\/, \/-, EitherT } object Platform { @@ -75,7 +74,7 @@ object Platform { p.future } - val artifact: Repository.Fetch[Task] = { artifact => + val artifact: Fetch.Content[Task] = { artifact => EitherT( Task { implicit ec => get(artifact.url) @@ -87,13 +86,18 @@ object Platform { ) } + implicit def fetch( + repositories: Seq[core.Repository] + ): Fetch.Metadata[Task] = + Fetch(repositories, Platform.artifact) + trait Logger { def fetching(url: String): Unit def fetched(url: String): Unit def other(url: String, msg: String): Unit } - def artifactWithLogger(logger: Logger): Repository.Fetch[Task] = { artifact => + def artifactWithLogger(logger: Logger): Fetch.Content[Task] = { artifact => EitherT( Task { implicit ec => Future(logger.fetching(artifact.url)) diff --git a/fetch-js/src/main/scala/coursier/Task.scala b/fetch-js/src/main/scala/coursier/Task.scala new file mode 100644 index 000000000..b1ab3aae9 --- /dev/null +++ b/fetch-js/src/main/scala/coursier/Task.scala @@ -0,0 +1,52 @@ +package coursier + +import scala.concurrent.{ ExecutionContext, Future } +import scalaz.{ Nondeterminism, Reducer } + +/** + * Minimal Future-based Task. + * + * Likely to be flawed and/or sub-optimal, but does the job. + */ +trait Task[T] { self => + def map[U](f: T => U): Task[U] = + new Task[U] { + def runF(implicit ec: ExecutionContext) = self.runF.map(f) + } + def flatMap[U](f: T => Task[U]): Task[U] = + new Task[U] { + def runF(implicit ec: ExecutionContext) = self.runF.flatMap(f(_).runF) + } + + def runF(implicit ec: ExecutionContext): Future[T] +} + +object Task { + def now[A](a: A): Task[A] = + new Task[A] { + def runF(implicit ec: ExecutionContext) = Future.successful(a) + } + def apply[A](f: ExecutionContext => Future[A]): Task[A] = + new Task[A] { + def runF(implicit ec: ExecutionContext) = f(ec) + } + def gatherUnordered[T](tasks: Seq[Task[T]], exceptionCancels: Boolean = false): Task[Seq[T]] = + new Task[Seq[T]] { + def runF(implicit ec: ExecutionContext) = Future.traverse(tasks)(_.runF) + } + + implicit val taskMonad: Nondeterminism[Task] = + new Nondeterminism[Task] { + def point[A](a: => A): Task[A] = Task.now(a) + def bind[A,B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f) + override def reduceUnordered[A, M](fs: Seq[Task[A]])(implicit R: Reducer[A, M]): Task[M] = + Task { implicit ec => + val f = Future.sequence(fs.map(_.runF)) + f.map { l => + (R.zero /: l)(R.snoc) + } + } + def chooseAny[A](head: Task[A], tail: Seq[Task[A]]): Task[(A, Seq[Task[A]])] = + ??? + } +} diff --git a/fetch-js/src/main/scala/scalaz/concurrent/package.scala b/fetch-js/src/main/scala/scalaz/concurrent/package.scala deleted file mode 100644 index 0a8a8591f..000000000 --- a/fetch-js/src/main/scala/scalaz/concurrent/package.scala +++ /dev/null @@ -1,42 +0,0 @@ -package scalaz - -import scala.concurrent.{ ExecutionContext, Future } - -/** Minimal Future-based Task */ -package object concurrent { - - trait Task[T] { self => - def map[U](f: T => U): Task[U] = - new Task[U] { - def runF(implicit ec: ExecutionContext) = self.runF.map(f) - } - def flatMap[U](f: T => Task[U]): Task[U] = - new Task[U] { - def runF(implicit ec: ExecutionContext) = self.runF.flatMap(f(_).runF) - } - - def runF(implicit ec: ExecutionContext): Future[T] - } - - object Task { - def now[A](a: A): Task[A] = - new Task[A] { - def runF(implicit ec: ExecutionContext) = Future.successful(a) - } - def apply[A](f: ExecutionContext => Future[A]): Task[A] = - new Task[A] { - def runF(implicit ec: ExecutionContext) = f(ec) - } - def gatherUnordered[T](tasks: Seq[Task[T]], exceptionCancels: Boolean = false): Task[Seq[T]] = - new Task[Seq[T]] { - def runF(implicit ec: ExecutionContext) = Future.traverse(tasks)(_.runF) - } - - implicit val taskMonad: Monad[Task] = - new Monad[Task] { - def point[A](a: => A): Task[A] = Task.now(a) - def bind[A,B](fa: Task[A])(f: A => Task[B]): Task[B] = fa.flatMap(f) - } - } - -} diff --git a/files/src/main/scala/coursier/Cache.scala b/files/src/main/scala/coursier/Cache.scala deleted file mode 100644 index 4f39c35e9..000000000 --- a/files/src/main/scala/coursier/Cache.scala +++ /dev/null @@ -1,134 +0,0 @@ -package coursier - -import java.io.{ File, PrintWriter } -import scala.io.Source - -object Cache { - - def mavenRepository(lines: Seq[String]): Option[MavenRepository] = { - def isMaven = - lines - .find(_.startsWith("maven:")) - .map(_.stripPrefix("maven:").trim) - .toSeq - .contains("true") - - def ivyLike = - lines - .find(_.startsWith("ivy-like:")) - .map(_.stripPrefix("ivy-like:").trim) - .toSeq - .contains("true") - - def base = - lines - .find(_.startsWith("base:")) - .map(_.stripPrefix("base:").trim) - .filter(_.nonEmpty) - - if (isMaven) - base.map(MavenRepository(_, ivyLike = ivyLike)) - else - None - } - - lazy val default = Cache(new File(sys.props("user.home") + "/.coursier/cache")) - -} - -case class Cache(cache: File) { - - import Cache._ - - lazy val repoDir = new File(cache, "repositories") - lazy val fileBase = new File(cache, "cache") - - lazy val defaultFile = new File(repoDir, ".default") - - def add(id: String, base: String, ivyLike: Boolean): Unit = { - repoDir.mkdirs() - val f = new File(repoDir, id) - val w = new PrintWriter(f) - try w.println((Seq("maven: true", s"base: $base") ++ (if (ivyLike) Seq("ivy-like: true") else Nil)).mkString("\n")) - finally w.close() - } - - def addCentral(): Unit = - add("central", "https://repo1.maven.org/maven2/", ivyLike = false) - - def addIvy2Local(): Unit = - add("ivy2local", new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, ivyLike = true) - - def init( - ifEmpty: Boolean = true, - verbose: Boolean = false - ): Unit = - if (!ifEmpty || !repoDir.exists() || !fileBase.exists()) { - if (verbose) - Console.err.println(s"Initializing $cache") - repoDir.mkdirs() - fileBase.mkdirs() - addCentral() - addIvy2Local() - setDefault("ivy2local", "central") - } - - def setDefault(ids: String*): Unit = { - defaultFile.getParentFile.mkdirs() - val w = new PrintWriter(defaultFile) - try w.println(ids.mkString("\n")) - finally w.close() - } - - def list(): Seq[(String, MavenRepository, (String, File))] = - Option(repoDir.listFiles()) - .map(_.toSeq) - .getOrElse(Nil) - .filter(f => f.isFile && !f.getName.startsWith(".")) - .flatMap { f => - val name = f.getName - val lines = Source.fromFile(f).getLines().toList - mavenRepository(lines).map(repo => - (name, repo, (repo.root, new File(fileBase, name))) - ) - } - - def map(): Map[String, (MavenRepository, (String, File))] = - list() - .map{case (id, repo, fileCache) => id -> (repo, fileCache) } - .toMap - - - def repositories(): Seq[MavenRepository] = - list().map(_._2) - - def repositoryMap(): Map[String, MavenRepository] = - list() - .map{case (id, repo, _) => id -> repo} - .toMap - - def fileCaches(): Seq[(String, File)] = - list().map(_._3) - - def default(withNotFound: Boolean = false): Seq[String] = - if (defaultFile.exists()) { - val default0 = - Source.fromFile(defaultFile) - .getLines() - .map(_.trim) - .filter(_.nonEmpty) - .toList - - val found = list() - .map(_._1) - .toSet - - default0 - .filter(found) - } else - Nil - - def files(): Files = - new Files(list().map{case (_, _, matching) => matching }, () => ???) - -} diff --git a/files/src/main/scala/coursier/CachePolicy.scala b/files/src/main/scala/coursier/CachePolicy.scala deleted file mode 100644 index 10c76392f..000000000 --- a/files/src/main/scala/coursier/CachePolicy.scala +++ /dev/null @@ -1,49 +0,0 @@ -package coursier - -import scalaz.\/ -import scalaz.concurrent.Task - -sealed trait CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] -} - -object CachePolicy { - def saving[E,T]( - remote: => Task[E \/ T] )( - save: T => Task[Unit] - ): Task[E \/ T] = { - for { - res <- remote - _ <- res.fold(_ => Task.now(()), t => save(t)) - } yield res - } - - case object Default extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res)) - } - case object LocalOnly extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local - } - case object ForceDownload extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - remote(None) - } -} diff --git a/files/src/main/scala/coursier/Fetch.scala b/files/src/main/scala/coursier/Fetch.scala deleted file mode 100644 index 2c0ab0e81..000000000 --- a/files/src/main/scala/coursier/Fetch.scala +++ /dev/null @@ -1,29 +0,0 @@ -package coursier - -import scalaz.concurrent.Task - -object Fetch { - - implicit def default( - repositories: Seq[core.Repository] - ): ResolutionProcess.Fetch[Task] = - apply(repositories, Platform.artifact) - - def apply( - repositories: Seq[core.Repository], - fetch: Repository.Fetch[Task], - extra: Repository.Fetch[Task]* - ): ResolutionProcess.Fetch[Task] = { - - modVers => Task.gatherUnordered( - modVers.map { case (module, version) => - def get(fetch: Repository.Fetch[Task]) = - Repository.find(repositories, module, version, fetch) - (get(fetch) /: extra)(_ orElse get(_)) - .run - .map((module, version) -> _) - } - ) - } - -} diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala deleted file mode 100644 index b7e6d7287..000000000 --- a/files/src/main/scala/coursier/Files.scala +++ /dev/null @@ -1,340 +0,0 @@ -package coursier - -import java.net.URL -import java.nio.channels.{ OverlappingFileLockException, FileLock } -import java.security.MessageDigest -import java.util.concurrent.{ Executors, ExecutorService } - -import scala.annotation.tailrec -import scalaz._ -import scalaz.concurrent.{ Task, Strategy } - -import java.io._ - -case class Files( - cache: Seq[(String, File)], - tmp: () => File, - concurrentDownloadCount: Int = Files.defaultConcurrentDownloadCount -) { - - lazy val defaultPool = - Executors.newFixedThreadPool(concurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) - - def withLocal(artifact: Artifact): Artifact = { - def local(url: String) = - if (url.startsWith("file:///")) - url.stripPrefix("file://") - else if (url.startsWith("file:/")) - url.stripPrefix("file:") - else - cache.find { case (base, _) => url.startsWith(base) } match { - case None => - // FIXME Means we were handed an artifact from repositories other than the known ones - println(cache.mkString("\n")) - println(url) - ??? - case Some((base, cacheDir)) => - cacheDir + "/" + url.stripPrefix(base) - } - - if (artifact.extra.contains("local")) - artifact - else - artifact.copy(extra = artifact.extra + ("local" -> - artifact.copy( - url = local(artifact.url), - checksumUrls = artifact.checksumUrls - .mapValues(local) - .toVector - .toMap, - extra = Map.empty - ) - )) - } - - def download( - artifact: Artifact, - withChecksums: Boolean = true, - logger: Option[Files.Logger] = None - )(implicit - cachePolicy: CachePolicy, - pool: ExecutorService = defaultPool - ): Task[Seq[((File, String), FileError \/ Unit)]] = { - val artifact0 = withLocal(artifact) - .extra - .getOrElse("local", artifact) - - val pairs = - Seq(artifact0.url -> artifact.url) ++ { - if (withChecksums) - (artifact0.checksumUrls.keySet intersect artifact.checksumUrls.keySet) - .toList - .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) - else - Nil - } - - - def locally(file: File, url: String) = - Task { - if (file.exists()) { - logger.foreach(_.foundLocally(url, file)) - \/-(file) - } else - -\/(FileError.NotFound(file.toString): FileError) - } - - // FIXME Things can go wrong here and are not properly handled, - // e.g. what if the connection gets closed during the transfer? - // (partial file on disk?) - def remote(file: File, url: String) = - Task { - try { - logger.foreach(_.downloadingArtifact(url)) - - val conn = new URL(url).openConnection() // FIXME Should this be closed? - // Dummy user-agent instead of the default "Java/...", - // so that we are not returned incomplete/erroneous metadata - // (Maven 2 compatibility? - happens for snapshot versioning metadata, - // this is SO FUCKING CRAZY) - conn.setRequestProperty("User-Agent", "") - - for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) - logger.foreach(_.downloadLength(url, len)) - - val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize) - - val result = - try { - file.getParentFile.mkdirs() - val out = new FileOutputStream(file) - try { - var lock: FileLock = null - try { - lock = out.getChannel.tryLock() - if (lock == null) - -\/(FileError.Locked(file.toString)) - else { - val b = Array.fill[Byte](Files.bufferSize)(0) - - @tailrec - def helper(count: Long): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - logger.foreach(_.downloadProgress(url, count + read)) - helper(count + read) - } - } - - helper(0L) - \/-(file) - } - } - catch { - case e: OverlappingFileLockException => - -\/(FileError.Locked(file.toString)) - } - finally if (lock != null) lock.release() - } finally out.close() - } finally in.close() - - for (lastModified <- Option(conn.getLastModified).filter(_ > 0L)) - file.setLastModified(lastModified) - - logger.foreach(_.downloadedArtifact(url, success = true)) - result - } - catch { case e: Exception => - logger.foreach(_.downloadedArtifact(url, success = false)) - -\/(FileError.DownloadError(e.getMessage)) - } - } - - - val tasks = - for ((f, url) <- pairs) yield { - val file = new File(f) - - if (url != ("file:" + f) && url != ("file://" + f)) { - assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url") - cachePolicy[FileError \/ File]( - _.isLeft )( - locally(file, url) )( - _ => remote(file, url) - ).map(e => (file, url) -> e.map(_ => ())) - } else - Task { - (file, url) -> { - if (file.exists()) - \/-(()) - else - -\/(FileError.NotFound(file.toString)) - } - } - } - - Nondeterminism[Task].gather(tasks) - } - - def validateChecksum( - artifact: Artifact, - sumType: String - )(implicit - pool: ExecutorService = defaultPool - ): Task[FileError \/ Unit] = { - val artifact0 = withLocal(artifact) - .extra - .getOrElse("local", artifact) - - - artifact0.checksumUrls.get(sumType) match { - case Some(sumFile) => - Task { - val sum = scala.io.Source.fromFile(sumFile) - .getLines() - .toStream - .headOption - .mkString - .takeWhile(!_.isSpaceChar) - - val md = MessageDigest.getInstance(sumType) - val is = new FileInputStream(new File(artifact0.url)) - try Files.withContent(is, md.update(_, 0, _)) - finally is.close() - - val digest = md.digest() - val calculatedSum = f"${BigInt(1, digest)}%040x" - - if (sum == calculatedSum) - \/-(()) - else - -\/(FileError.WrongChecksum(sumType, calculatedSum, sum, artifact0.url, sumFile)) - } - - case None => - Task.now(-\/(FileError.ChecksumNotFound(sumType, artifact0.url))) - } - } - - def file( - artifact: Artifact, - checksum: Option[String] = Some("SHA-1"), - logger: Option[Files.Logger] = None - )(implicit - cachePolicy: CachePolicy, - pool: ExecutorService = defaultPool - ): EitherT[Task, FileError, File] = - EitherT { - val res = download(artifact, withChecksums = checksum.nonEmpty, logger = logger).map { - results => - val ((f, _), res) = results.head - res.map(_ => f) - } - - checksum.fold(res) { sumType => - res.flatMap { - case err @ -\/(_) => Task.now(err) - case \/-(f) => - validateChecksum(artifact, sumType) - .map(_.map(_ => f)) - } - } - } - - def fetch( - checksum: Option[String] = Some("SHA-1"), - logger: Option[Files.Logger] = None - )(implicit - cachePolicy: CachePolicy, - pool: ExecutorService = defaultPool - ): Repository.Fetch[Task] = { - artifact => - file(artifact, checksum = checksum, logger = logger)(cachePolicy).leftMap(_.message).map { f => - // FIXME Catch error here? - scala.io.Source.fromFile(f)("UTF-8").mkString - } - } - -} - -object Files { - - lazy val ivy2Local = MavenRepository( - new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, - ivyLike = true - ) - - val defaultConcurrentDownloadCount = 6 - - trait Logger { - def foundLocally(url: String, f: File): Unit = {} - def downloadingArtifact(url: String): Unit = {} - def downloadLength(url: String, length: Long): Unit = {} - def downloadProgress(url: String, downloaded: Long): Unit = {} - def downloadedArtifact(url: String, success: Boolean): Unit = {} - } - - var bufferSize = 1024*1024 - - def readFullySync(is: InputStream) = { - val buffer = new ByteArrayOutputStream() - val data = Array.ofDim[Byte](16384) - - var nRead = is.read(data, 0, data.length) - while (nRead != -1) { - buffer.write(data, 0, nRead) - nRead = is.read(data, 0, data.length) - } - - buffer.flush() - buffer.toByteArray - } - - def readFully(is: => InputStream) = - Task { - \/.fromTryCatchNonFatal { - val is0 = is - val b = - try readFullySync(is0) - finally is0.close() - - new String(b, "UTF-8") - } .leftMap(_.getMessage) - } - - def withContent(is: InputStream, f: (Array[Byte], Int) => Unit): Unit = { - val data = Array.ofDim[Byte](16384) - - var nRead = is.read(data, 0, data.length) - while (nRead != -1) { - f(data, nRead) - nRead = is.read(data, 0, data.length) - } - } - -} - -sealed trait FileError { - def message: String -} - -object FileError { - - case class DownloadError(message0: String) extends FileError { - def message = s"Download error: $message0" - } - case class NotFound(file: String) extends FileError { - def message = s"$file: not found" - } - case class Locked(file: String) extends FileError { - def message = s"$file: locked" - } - case class ChecksumNotFound(sumType: String, file: String) extends FileError { - def message = s"$file: $sumType checksum not found" - } - case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError { - def message = s"$file: $sumType checksum validation failed" - } - -} diff --git a/plugin/src/main/scala/coursier/CoursierPlugin.scala b/plugin/src/main/scala/coursier/CoursierPlugin.scala new file mode 100644 index 000000000..7d617f7f4 --- /dev/null +++ b/plugin/src/main/scala/coursier/CoursierPlugin.scala @@ -0,0 +1,52 @@ +package coursier + +import java.io.File + +import sbt._ +import sbt.Keys._ + +object CoursierPlugin extends AutoPlugin { + + override def trigger = allRequirements + + override def requires = sbt.plugins.IvyPlugin + + object autoImport { + val coursierParallelDownloads = Keys.coursierParallelDownloads + val coursierMaxIterations = Keys.coursierMaxIterations + val coursierChecksums = Keys.coursierChecksums + val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums + val coursierCachePolicy = Keys.coursierCachePolicy + val coursierVerbosity = Keys.coursierVerbosity + val coursierResolvers = Keys.coursierResolvers + val coursierSbtResolvers = Keys.coursierSbtResolvers + val coursierCache = Keys.coursierCache + val coursierProject = Keys.coursierProject + val coursierProjects = Keys.coursierProjects + val coursierPublications = Keys.coursierPublications + val coursierSbtClassifiersModule = Keys.coursierSbtClassifiersModule + } + + import autoImport._ + + + override lazy val projectSettings = Seq( + coursierParallelDownloads := 6, + coursierMaxIterations := 50, + coursierChecksums := Seq(Some("SHA-1"), None), + coursierArtifactsChecksums := Seq(None), + coursierCachePolicy := CachePolicy.FetchMissing, + coursierVerbosity := 1, + coursierResolvers <<= Tasks.coursierResolversTask, + coursierSbtResolvers <<= externalResolvers in updateSbtClassifiers, + coursierCache := new File(sys.props("user.home") + "/.coursier/sbt"), + update <<= Tasks.updateTask(withClassifiers = false), + updateClassifiers <<= Tasks.updateTask(withClassifiers = true), + updateSbtClassifiers in Defaults.TaskGlobal <<= Tasks.updateTask(withClassifiers = true, sbtClassifiers = true), + coursierProject <<= Tasks.coursierProjectTask, + coursierProjects <<= Tasks.coursierProjectsTask, + coursierPublications <<= Tasks.coursierPublicationsTask, + coursierSbtClassifiersModule <<= classifiersModule in updateSbtClassifiers + ) + +} diff --git a/plugin/src/main/scala/coursier/FromSbt.scala b/plugin/src/main/scala/coursier/FromSbt.scala new file mode 100644 index 000000000..2a1b653b1 --- /dev/null +++ b/plugin/src/main/scala/coursier/FromSbt.scala @@ -0,0 +1,140 @@ +package coursier + +import coursier.ivy.{ IvyXml, IvyRepository } +import sbt.mavenint.SbtPomExtraProperties +import sbt.{ Resolver, CrossVersion, ModuleID } + +object FromSbt { + + def sbtModuleIdName( + moduleId: ModuleID, + scalaVersion: => String, + scalaBinaryVersion: => String + ): String = + sbtCrossVersionName(moduleId.name, moduleId.crossVersion, scalaVersion, scalaBinaryVersion) + + def sbtCrossVersionName( + name: String, + crossVersion: CrossVersion, + scalaVersion: => String, + scalaBinaryVersion: => String + ): String = crossVersion match { + case CrossVersion.Disabled => name + case f: CrossVersion.Full => name + "_" + f.remapVersion(scalaVersion) + case f: CrossVersion.Binary => name + "_" + f.remapVersion(scalaBinaryVersion) + } + + def attributes(attr: Map[String, String]): Map[String, String] = + attr.map { case (k, v) => + k.stripPrefix("e:") -> v + }.filter { case (k, _) => + !k.startsWith(SbtPomExtraProperties.POM_INFO_KEY_PREFIX) + } + + def dependencies( + module: ModuleID, + scalaVersion: String, + scalaBinaryVersion: String + ): Seq[(String, Dependency)] = { + + // TODO Warn about unsupported properties in `module` + + val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) + + val dep = Dependency( + Module(module.organization, fullName, FromSbt.attributes(module.extraDependencyAttributes)), + module.revision, + exclusions = module.exclusions.map { rule => + // FIXME Other `rule` fields are ignored here + (rule.organization, rule.name) + }.toSet + ) + + val mapping = module.configurations.getOrElse("compile") + val allMappings = IvyXml.mappings(mapping) + + val attributes = + if (module.explicitArtifacts.isEmpty) + Seq(Attributes()) + else + module.explicitArtifacts.map { a => + Attributes(`type` = a.extension, classifier = a.classifier.getOrElse("")) + } + + for { + (from, to) <- allMappings.toSeq + attr <- attributes + } yield from -> dep.copy(configuration = to, attributes = attr) + } + + def project( + projectID: ModuleID, + allDependencies: Seq[ModuleID], + ivyConfigurations: Map[String, Seq[String]], + scalaVersion: String, + scalaBinaryVersion: String + ): Project = { + + // FIXME Ignored for now - easy to support though + // val sbtDepOverrides = dependencyOverrides.value + // val sbtExclusions = excludeDependencies.value + + val deps = allDependencies.flatMap(dependencies(_, scalaVersion, scalaBinaryVersion)) + + Project( + Module( + projectID.organization, + sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion), + FromSbt.attributes(projectID.extraDependencyAttributes) + ), + projectID.revision, + deps, + ivyConfigurations, + None, + Nil, + Nil, + Nil, + None, + None, + Nil, + Info.empty + ) + } + + def repository(resolver: Resolver, ivyProperties: Map[String, String]): Option[Repository] = + resolver match { + case sbt.MavenRepository(_, root) => + if (root.startsWith("http://") || root.startsWith("https://")) { + val root0 = if (root.endsWith("/")) root else root + "/" + Some(MavenRepository(root0, sbtAttrStub = true)) + } else { + Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it") + None + } + + case sbt.FileRepository(_, _, patterns) + if patterns.ivyPatterns.lengthCompare(1) == 0 && + patterns.ivyPatterns == patterns.artifactPatterns => + + Some(IvyRepository( + "file://" + patterns.ivyPatterns.head, + changing = Some(true), + properties = ivyProperties + )) + + case sbt.URLRepository(_, patterns) + if patterns.ivyPatterns.lengthCompare(1) == 0 && + patterns.ivyPatterns == patterns.artifactPatterns => + + Some(IvyRepository( + patterns.ivyPatterns.head, + changing = None, + properties = ivyProperties + )) + + case other => + Console.err.println(s"Warning: unrecognized repository ${other.name}, ignoring it") + None + } + +} diff --git a/plugin/src/main/scala/coursier/InterProjectRepository.scala b/plugin/src/main/scala/coursier/InterProjectRepository.scala new file mode 100644 index 000000000..8eb094420 --- /dev/null +++ b/plugin/src/main/scala/coursier/InterProjectRepository.scala @@ -0,0 +1,27 @@ +package coursier + +import scalaz.{ -\/, \/-, Monad, EitherT } + +case class InterProjectRepository(projects: Seq[Project]) extends Repository { + + private val map = projects + .map { proj => proj.moduleVersion -> proj } + .toMap + + def find[F[_]]( + module: Module, + version: String, + fetch: Fetch.Content[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, (Artifact.Source, Project)] = { + val res = map.get((module, version)) match { + case Some(proj) => + \/-((Artifact.Source.empty, proj)) + case None => + -\/("Not found") + } + + EitherT(F.point(res)) + } +} \ No newline at end of file diff --git a/plugin/src/main/scala/coursier/Keys.scala b/plugin/src/main/scala/coursier/Keys.scala new file mode 100644 index 000000000..d870f320d --- /dev/null +++ b/plugin/src/main/scala/coursier/Keys.scala @@ -0,0 +1,26 @@ +package coursier + +import java.io.File +import coursier.core.Publication +import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey } + +object Keys { + val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads", "") + val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations", "") + val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums", "") + val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums", "") + val coursierCachePolicy = SettingKey[CachePolicy]("coursier-cache-policy", "") + + val coursierVerbosity = SettingKey[Int]("coursier-verbosity", "") + + val coursierResolvers = TaskKey[Seq[Resolver]]("coursier-resolvers", "") + val coursierSbtResolvers = TaskKey[Seq[Resolver]]("coursier-sbt-resolvers", "") + + val coursierCache = SettingKey[File]("coursier-cache", "") + + val coursierProject = TaskKey[Project]("coursier-project", "") + val coursierProjects = TaskKey[Seq[Project]]("coursier-projects", "") + val coursierPublications = TaskKey[Seq[(String, Publication)]]("coursier-publications", "") + + val coursierSbtClassifiersModule = TaskKey[GetClassifiersModule]("coursier-sbt-classifiers-module", "") +} diff --git a/plugin/src/main/scala/coursier/MakeIvyXml.scala b/plugin/src/main/scala/coursier/MakeIvyXml.scala new file mode 100644 index 000000000..405e69af4 --- /dev/null +++ b/plugin/src/main/scala/coursier/MakeIvyXml.scala @@ -0,0 +1,64 @@ +package coursier + +import scala.xml.{ Node, PrefixedAttribute } + +object MakeIvyXml { + + def apply(project: Project): Node = { + + val baseInfoAttrs = .attributes + + val infoAttrs = project.module.attributes.foldLeft(baseInfoAttrs) { + case (acc, (k, v)) => + new PrefixedAttribute("e", k, v, acc) + } + + val licenseElems = project.info.licenses.map { + case (name, urlOpt) => + var n = + for (url <- urlOpt) + n = n % .attributes + n + } + + val infoElem = { + + {licenseElems} + {project.info.description} + + } % infoAttrs + + val confElems = project.configurations.toVector.map { + case (name, extends0) => + var n = + if (extends0.nonEmpty) + n = n % .attributes + n + } + + val publicationElems = project.publications.map { + case (conf, pub) => + var n = + if (pub.classifier.nonEmpty) + n = n % .attributes + n + } + + val dependencyElems = project.dependencies.toVector.map { + case (conf, dep) => + ${dep.configuration}"} /> + } + + + {infoElem} + {confElems} + {publicationElems} + {dependencyElems} + + } + +} diff --git a/plugin/src/main/scala/coursier/Structure.scala b/plugin/src/main/scala/coursier/Structure.scala new file mode 100644 index 000000000..c9e5ea7e3 --- /dev/null +++ b/plugin/src/main/scala/coursier/Structure.scala @@ -0,0 +1,40 @@ +package coursier + +import sbt._ + +// things from sbt-structure +object Structure { + import Def.Initialize._ + + def structure(state: State): Load.BuildStructure = + sbt.Project.structure(state) + + implicit def `enrich SettingKey`[T](key: SettingKey[T]) = new { + def find(state: State): Option[T] = + key.get(structure(state).data) + + def get(state: State): T = + find(state).get + + def getOrElse(state: State, default: => T): T = + find(state).getOrElse(default) + } + + implicit def `enrich TaskKey`[T](key: TaskKey[T]) = new { + def find(state: State): Option[sbt.Task[T]] = + key.get(structure(state).data) + + def get(state: State): sbt.Task[T] = + find(state).get + + def forAllProjects(state: State, projects: Seq[ProjectRef]): sbt.Task[Map[ProjectRef, T]] = { + val tasks = projects.flatMap(p => key.in(p).get(structure(state).data).map(_.map(it => (p, it)))) + std.TaskExtra.joinTasks(tasks).join.map(_.toMap) + } + + def forAllConfigurations(state: State, configurations: Seq[sbt.Configuration]): sbt.Task[Map[sbt.Configuration, T]] = { + val tasks = configurations.flatMap(c => key.in(c).get(structure(state).data).map(_.map(it => (c, it)))) + std.TaskExtra.joinTasks(tasks).join.map(_.toMap) + } + } +} diff --git a/plugin/src/main/scala/coursier/Tasks.scala b/plugin/src/main/scala/coursier/Tasks.scala new file mode 100644 index 000000000..1e43da038 --- /dev/null +++ b/plugin/src/main/scala/coursier/Tasks.scala @@ -0,0 +1,436 @@ +package coursier + +import java.io.{ OutputStreamWriter, File } +import java.nio.file.Files +import java.util.concurrent.Executors + +import coursier.core.Publication +import coursier.ivy.IvyRepository +import coursier.Keys._ +import coursier.Structure._ +import org.apache.ivy.core.module.id.ModuleRevisionId + +import sbt.{ UpdateReport, Classpaths, Resolver, Def } +import sbt.Configurations.{ Compile, Test } +import sbt.Keys._ + +import scala.collection.mutable +import scala.collection.JavaConverters._ + +import scalaz.{ \/-, -\/ } +import scalaz.concurrent.{ Task, Strategy } + +object Tasks { + + def coursierResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = Def.task { + var resolvers = externalResolvers.value + if (sbtPlugin.value) + resolvers = Seq( + sbtResolver.value, + Classpaths.sbtPluginReleases + ) ++ resolvers + resolvers + } + + def coursierProjectTask: Def.Initialize[sbt.Task[Project]] = + ( + sbt.Keys.state, + sbt.Keys.thisProjectRef + ).flatMap { (state, projectRef) => + + // should projectID.configurations be used instead? + val configurations = ivyConfigurations.in(projectRef).get(state) + + val allDependenciesTask = allDependencies.in(projectRef).get(state) + + for { + allDependencies <- allDependenciesTask + } yield { + + FromSbt.project( + projectID.in(projectRef).get(state), + allDependencies, + configurations.map { cfg => cfg.name -> cfg.extendsConfigs.map(_.name) }.toMap, + scalaVersion.in(projectRef).get(state), + scalaBinaryVersion.in(projectRef).get(state) + ) + } + } + + def coursierProjectsTask: Def.Initialize[sbt.Task[Seq[Project]]] = + sbt.Keys.state.flatMap { state => + val projects = structure(state).allProjectRefs + coursierProject.forAllProjects(state, projects).map(_.values.toVector) + } + + def coursierPublicationsTask: Def.Initialize[sbt.Task[Seq[(String, Publication)]]] = + ( + sbt.Keys.state, + sbt.Keys.thisProjectRef, + sbt.Keys.projectID, + sbt.Keys.scalaVersion, + sbt.Keys.scalaBinaryVersion + ).map { (state, projectRef, projId, sv, sbv) => + + val packageTasks = Seq(packageBin, packageSrc, packageDoc) + val configs = Seq(Compile, Test) + + val sbtArtifacts = + for { + pkgTask <- packageTasks + config <- configs + } yield { + val publish = publishArtifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, false) + if (publish) + Option(artifact.in(projectRef).in(pkgTask).in(config).getOrElse(state, null)) + .map(config.name -> _) + else + None + } + + sbtArtifacts.collect { + case Some((config, artifact)) => + val name = FromSbt.sbtCrossVersionName( + artifact.name, + projId.crossVersion, + sv, + sbv + ) + + val publication = Publication( + name, + artifact.`type`, + artifact.extension, + artifact.classifier.getOrElse("") + ) + + config -> publication + } + } + + // FIXME More things should possibly be put here too (resolvers, etc.) + private case class CacheKey( + resolution: Resolution, + withClassifiers: Boolean, + sbtClassifiers: Boolean + ) + + private val resolutionsCache = new mutable.HashMap[CacheKey, UpdateReport] + + def updateTask(withClassifiers: Boolean, sbtClassifiers: Boolean = false) = Def.task { + + // SBT logging should be better than that most of the time... + def errPrintln(s: String): Unit = scala.Console.err.println(s) + + def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = + map.groupBy { case (k, _) => k }.map { + case (k, l) => + k -> l.map { case (_, v) => v } + } + + // let's update only one module at once, for a better output + // Downloads are already parallel, no need to parallelize further anyway + synchronized { + + lazy val cm = coursierSbtClassifiersModule.value + + val currentProject = + if (sbtClassifiers) + FromSbt.project( + cm.id, + cm.modules, + cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap, + scalaVersion.value, + scalaBinaryVersion.value + ) + else { + val proj = coursierProject.value + val publications = coursierPublications.value + proj.copy(publications = publications) + } + + val ivySbt0 = ivySbt.value + val ivyCacheManager = ivySbt0.withIvy(streams.value.log)(ivy => + ivy.getResolutionCacheManager + ) + + val ivyModule = ModuleRevisionId.newInstance( + currentProject.module.organization, + currentProject.module.name, + currentProject.version, + currentProject.module.attributes.asJava + ) + val cacheIvyFile = ivyCacheManager.getResolvedIvyFileInCache(ivyModule) + val cacheIvyPropertiesFile = ivyCacheManager.getResolvedIvyPropertiesInCache(ivyModule) + + val projects = coursierProjects.value + + val parallelDownloads = coursierParallelDownloads.value + val checksums = coursierChecksums.value + val artifactsChecksums = coursierArtifactsChecksums.value + val maxIterations = coursierMaxIterations.value + val cachePolicy = coursierCachePolicy.value + val cacheDir = coursierCache.value + + val resolvers = + if (sbtClassifiers) + coursierSbtResolvers.value + else + coursierResolvers.value + + val verbosity = coursierVerbosity.value + + + val startRes = Resolution( + currentProject.dependencies.map { case (_, dep) => dep }.toSet, + filter = Some(dep => !dep.optional), + forceVersions = projects.map(_.moduleVersion).toMap + ) + + // required for publish to be fine, later on + def writeIvyFiles() = { + val printer = new scala.xml.PrettyPrinter(80, 2) + + val b = new StringBuilder + b ++= """""" + b += '\n' + b ++= printer.format(MakeIvyXml(currentProject)) + cacheIvyFile.getParentFile.mkdirs() + Files.write(cacheIvyFile.toPath, b.result().getBytes("UTF-8")) + + // Just writing an empty file here... Are these only used? + cacheIvyPropertiesFile.getParentFile.mkdirs() + Files.write(cacheIvyPropertiesFile.toPath, "".getBytes("UTF-8")) + } + + def report = { + if (verbosity >= 1) { + println("InterProjectRepository") + for (p <- projects) + println(s" ${p.module}:${p.version}") + } + + val globalPluginsRepo = IvyRepository( + new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString + + "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]", + withChecksums = false, + withSignatures = false, + withArtifacts = false + ) + + val interProjectRepo = InterProjectRepository(projects) + + val ivyProperties = Map( + "ivy.home" -> s"${sys.props("user.home")}/.ivy2" + ) ++ sys.props + + val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties)) + + val caches = Seq( + "http://" -> new File(cacheDir, "http"), + "https://" -> new File(cacheDir, "https") + ) + + val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) + + def createLogger() = new TermDisplay( + new OutputStreamWriter(System.err), + fallbackMode = sys.env.get("COURSIER_NO_TERM").nonEmpty + ) + + val resLogger = createLogger() + + val fetch = coursier.Fetch( + repositories, + Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool), + Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool) + ) + + def depsRepr(deps: Seq[(String, Dependency)]) = + deps.map { case (config, dep) => + s"${dep.module}:${dep.version}:$config->${dep.configuration}" + }.sorted.distinct + + def depsRepr0(deps: Seq[Dependency]) = + deps.map { dep => + s"${dep.module}:${dep.version}:${dep.configuration}" + }.sorted.distinct + + if (verbosity >= 1) { + errPrintln(s"Repositories:") + val repositories0 = repositories.map { + case r: IvyRepository => r.copy(properties = Map.empty) + case r: InterProjectRepository => r.copy(projects = Nil) + case r => r + } + for (repo <- repositories0) + errPrintln(s" $repo") + } + + if (verbosity >= 0) + errPrintln(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") + if (verbosity >= 1) + for (depRepr <- depsRepr(currentProject.dependencies)) + errPrintln(s" $depRepr") + + resLogger.init() + + val res = startRes + .process + .run(fetch, maxIterations) + .attemptRun + .leftMap(ex => throw new Exception(s"Exception during resolution", ex)) + .merge + + resLogger.stop() + + + if (!res.isDone) + throw new Exception(s"Maximum number of iteration reached!") + + if (verbosity >= 0) + errPrintln("Resolution done") + if (verbosity >= 1) + for (depRepr <- depsRepr0(res.minDependencies.toSeq)) + errPrintln(s" $depRepr") + + def repr(dep: Dependency) = { + // dep.version can be an interval, whereas the one from project can't + val version = res + .projectCache + .get(dep.moduleVersion) + .map(_._2.version) + .getOrElse(dep.version) + val extra = + if (version == dep.version) "" + else s" ($version for ${dep.version})" + + ( + Seq( + dep.module.organization, + dep.module.name, + dep.attributes.`type` + ) ++ + Some(dep.attributes.classifier) + .filter(_.nonEmpty) + .toSeq ++ + Seq( + version + ) + ).mkString(":") + extra + } + + if (res.conflicts.nonEmpty) { + // Needs test + println(s"${res.conflicts.size} conflict(s):\n ${res.conflicts.toList.map(repr).sorted.mkString(" \n")}") + } + + val errors = res.errors + + if (errors.nonEmpty) { + println(s"\n${errors.size} error(s):") + for ((dep, errs) <- errors) { + println(s" ${dep.module}:${dep.version}:\n${errs.map(" " + _.replace("\n", " \n")).mkString("\n")}") + } + throw new Exception(s"Encountered ${errors.length} error(s)") + } + + val classifiers = + if (withClassifiers) + Some { + if (sbtClassifiers) + cm.classifiers + else + transitiveClassifiers.value + } + else + None + + val allArtifacts = + classifiers match { + case None => res.artifacts + case Some(cl) => res.classifiersArtifacts(cl) + } + + val artifactsLogger = createLogger() + + val artifactFileOrErrorTasks = allArtifacts.toVector.map { a => + Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _)) + } + + if (verbosity >= 0) + errPrintln(s"Fetching artifacts") + + artifactsLogger.init() + + val artifactFilesOrErrors = Task.gatherUnordered(artifactFileOrErrorTasks).attemptRun match { + case -\/(ex) => + throw new Exception(s"Error while downloading / verifying artifacts", ex) + case \/-(l) => + l.toMap + } + + artifactsLogger.stop() + + if (verbosity >= 0) + errPrintln(s"Fetching artifacts: done") + + val configs = { + val configs0 = ivyConfigurations.value.map { config => + config.name -> config.extendsConfigs.map(_.name) + }.toMap + + def allExtends(c: String) = { + // possibly bad complexity + def helper(current: Set[String]): Set[String] = { + val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) + if ((newSet -- current).nonEmpty) + helper(newSet) + else + newSet + } + + helper(Set(c)) + } + + configs0.map { + case (config, _) => + config -> allExtends(config) + } + } + + def artifactFileOpt(artifact: Artifact) = { + val fileOrError = artifactFilesOrErrors.getOrElse(artifact, -\/("Not downloaded")) + + fileOrError match { + case \/-(file) => + if (file.toString.contains("file:/")) + throw new Exception(s"Wrong path: $file") + Some(file) + case -\/(err) => + errPrintln(s"${artifact.url}: $err") + None + } + } + + val depsByConfig = grouped(currentProject.dependencies) + + writeIvyFiles() + + ToSbt.updateReport( + depsByConfig, + res, + configs, + classifiers, + artifactFileOpt + ) + } + + resolutionsCache.getOrElseUpdate( + CacheKey(startRes.copy(filter = None), withClassifiers, sbtClassifiers), + report + ) + } + } + +} diff --git a/plugin/src/main/scala/coursier/ToSbt.scala b/plugin/src/main/scala/coursier/ToSbt.scala new file mode 100644 index 000000000..8839f8ffa --- /dev/null +++ b/plugin/src/main/scala/coursier/ToSbt.scala @@ -0,0 +1,179 @@ +package coursier + +import java.util.GregorianCalendar + +import sbt._ + +object ToSbt { + + def moduleId(dependency: Dependency): sbt.ModuleID = + sbt.ModuleID( + dependency.module.organization, + dependency.module.name, + dependency.version, + configurations = Some(dependency.configuration), + extraAttributes = dependency.module.attributes + ) + + def artifact(module: Module, artifact: Artifact): sbt.Artifact = + sbt.Artifact( + module.name, + artifact.attributes.`type`, + "jar", + Some(artifact.attributes.classifier).filter(_.nonEmpty), + Nil, + Some(url(artifact.url)), + Map.empty + ) + + def moduleReport( + dependency: Dependency, + dependees: Seq[(Dependency, Project)], + project: Project, + artifacts: Seq[(Artifact, Option[File])] + ): sbt.ModuleReport = { + + val sbtArtifacts = artifacts.collect { + case (artifact, Some(file)) => + (ToSbt.artifact(dependency.module, artifact), file) + } + val sbtMissingArtifacts = artifacts.collect { + case (artifact, None) => + ToSbt.artifact(dependency.module, artifact) + } + + val publicationDate = project.info.publication.map { dt => + new GregorianCalendar(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second).getTime + } + + val callers = dependees.map { + case (dependee, dependeeProj) => + new Caller( + ToSbt.moduleId(dependee), + dependeeProj.configurations.keys.toVector, + dependee.module.attributes ++ dependeeProj.properties, + // FIXME Set better values here + isForceDependency = false, + isChangingDependency = false, + isTransitiveDependency = false, + isDirectlyForceDependency = false + ) + } + + new sbt.ModuleReport( + module = ToSbt.moduleId(dependency), + artifacts = sbtArtifacts, + missingArtifacts = sbtMissingArtifacts, + status = None, + publicationDate = publicationDate, + resolver = None, + artifactResolver = None, + evicted = false, + evictedData = None, + evictedReason = None, + problem = None, + homepage = Some(project.info.homePage).filter(_.nonEmpty), + extraAttributes = dependency.module.attributes ++ project.properties, + isDefault = None, + branch = None, + configurations = project.configurations.keys.toVector, + licenses = project.info.licenses, + callers = callers + ) + } + + private def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = + map.groupBy { case (k, _) => k }.map { + case (k, l) => + k -> l.map { case (_, v) => v } + } + + def moduleReports( + res: Resolution, + classifiersOpt: Option[Seq[String]], + artifactFileOpt: Artifact => Option[File] + ) = { + val depArtifacts = + classifiersOpt match { + case None => res.dependencyArtifacts + case Some(cl) => res.dependencyClassifiersArtifacts(cl) + } + + val groupedDepArtifacts = grouped(depArtifacts) + + val versions = res.dependencies.toVector.map { dep => + dep.module -> dep.version + }.toMap + + def clean(dep: Dependency): Dependency = + dep.copy(configuration = "", exclusions = Set.empty, optional = false) + + val reverseDependencies = res.reverseDependencies + .toVector + .map { case (k, v) => + clean(k) -> v.map(clean) + } + .groupBy { case (k, v) => k } + .mapValues { v => + v.flatMap { + case (_, l) => l + } + } + .toVector + .toMap + + groupedDepArtifacts.map { + case (dep, artifacts) => + val (_, proj) = res.projectCache(dep.moduleVersion) + + // FIXME Likely flaky... + val dependees = reverseDependencies + .getOrElse(clean(dep.copy(version = "")), Vector.empty) + .map { dependee0 => + val version = versions(dependee0.module) + val dependee = dependee0.copy(version = version) + val (_, dependeeProj) = res.projectCache(dependee.moduleVersion) + (dependee, dependeeProj) + } + + ToSbt.moduleReport( + dep, + dependees, + proj, + artifacts.map(a => a -> artifactFileOpt(a)) + ) + } + } + + def updateReport( + configDependencies: Map[String, Seq[Dependency]], + resolution: Resolution, + configs: Map[String, Set[String]], + classifiersOpt: Option[Seq[String]], + artifactFileOpt: Artifact => Option[File] + ): sbt.UpdateReport = { + + val configReports = configs.map { + case (config, extends0) => + val configDeps = extends0.flatMap(configDependencies.getOrElse(_, Nil)) + val subRes = resolution.subset(configDeps) + + val reports = ToSbt.moduleReports(subRes, classifiersOpt, artifactFileOpt) + + new ConfigurationReport( + config, + reports.toVector, + Nil, + Nil + ) + } + + new UpdateReport( + null, + configReports.toVector, + new UpdateStats(-1L, -1L, -1L, cached = false), + Map.empty + ) + } + +} diff --git a/project/plugins.sbt b/project/plugins.sbt index d26d5c6bc..ad7b57ef9 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,3 +3,4 @@ addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.5") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") addSbtPlugin("com.github.gseitz" % "sbt-release" % "0.8.5") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.1.0") +addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.4.0") diff --git a/project/travis.sh b/project/travis.sh index 25fddad7d..0b0fddb5e 100755 --- a/project/travis.sh +++ b/project/travis.sh @@ -29,21 +29,26 @@ function isMasterOrDevelop() { # web sub-project doesn't compile in 2.10 (no scalajs-react) if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then - SBT_COMMANDS="cli/compile" + IS_210=1 + SBT_COMMANDS="bootstrap/compile coreJVM/compile coreJS/compile cache/compile web/compile testsJVM/test testsJS/test" else - SBT_COMMANDS="compile" + IS_210=0 + SBT_COMMANDS="compile test" fi # Required for ~/.ivy2/local repo tests ~/sbt coreJVM/publish-local -SBT_COMMANDS="$SBT_COMMANDS test" - # TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed PUSH_GHPAGES=0 if isNotPr && publish && isMaster; then - SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish files/publish cli/publish" + SBT_COMMANDS="$SBT_COMMANDS coreJVM/publish coreJS/publish cache/publish" + if [ "$IS_210" = 1 ]; then + SBT_COMMANDS="$SBT_COMMANDS plugin/publish" + else + SBT_COMMANDS="$SBT_COMMANDS cli/publish" + fi fi if isNotPr && publish && isMasterOrDevelop; then diff --git a/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala index ac0c31865..db4006289 100644 --- a/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.{ Module, Files } +import coursier.{ Module, Cache } import utest._ object IvyLocalTests extends TestSuite { @@ -9,8 +9,8 @@ object IvyLocalTests extends TestSuite { 'coursier{ // Assume this module (and the sub-projects it depends on) is published locally CentralTests.resolutionCheck( - Module("com.github.alexarchambault", "coursier_2.11"), "0.1.0-SNAPSHOT", - Some(Files.ivy2Local)) + Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-SNAPSHOT", + Some(Cache.ivy2Local)) } } diff --git a/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT similarity index 75% rename from tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT rename to tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT index cc4c8e9b1..58ba0a4fc 100644 --- a/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT +++ b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/1.0.0-SNAPSHOT @@ -1,4 +1,4 @@ -com.github.alexarchambault:coursier_2.11:jar:0.1.0-SNAPSHOT +com.github.alexarchambault:coursier_2.11:jar:1.0.0-SNAPSHOT org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4 org.scala-lang.modules:scala-xml_2.11:jar:1.0.4 org.scala-lang:scala-library:jar:2.11.7 diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index 751397d13..f8b4b85fc 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -4,7 +4,7 @@ package test import utest._ import scala.async.Async.{ async, await } -import coursier.Fetch.default +import coursier.Platform.fetch import coursier.test.compatibility._ object CentralTests extends TestSuite { @@ -44,7 +44,8 @@ object CentralTests extends TestSuite { def resolutionCheck( module: Module, version: String, - extraRepo: Option[Repository] = None + extraRepo: Option[Repository] = None, + configuration: String = "" ) = async { val expected = @@ -54,7 +55,7 @@ object CentralTests extends TestSuite { .split('\n') .toSeq - val dep = Dependency(module, version) + val dep = Dependency(module, version, configuration = configuration) val res = await(resolve(Set(dep), extraRepo = extraRepo)) val result = res @@ -138,6 +139,7 @@ object CentralTests extends TestSuite { resolutionCheck( Module("com.github.fommil", "java-logging"), "1.2-SNAPSHOT", + configuration = "runtime", extraRepo = Some(MavenRepository("https://oss.sonatype.org/content/repositories/public/")) ) } diff --git a/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala index 6bea7b696..bb8452b79 100644 --- a/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala +++ b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala @@ -21,7 +21,7 @@ object PomParsingTests extends TestSuite { """ - val expected = \/-(Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra"))) + val expected = \/-("" -> Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra"))) val result = Pom.dependency(xmlParse(depNode).right.get) @@ -90,7 +90,7 @@ object PomParsingTests extends TestSuite { None, Profile.Activation(Nil), Seq( - Dependency(Module("comp", "lib"), "0.2")), + "" -> Dependency(Module("comp", "lib"), "0.2")), Nil, Map.empty )) @@ -122,7 +122,7 @@ object PomParsingTests extends TestSuite { Profile.Activation(Nil), Nil, Seq( - Dependency(Module("comp", "lib"), "0.2", scope = Scope.Test)), + "test" -> Dependency(Module("comp", "lib"), "0.2")), Map.empty )) @@ -194,7 +194,7 @@ object PomParsingTests extends TestSuite { val node = parsed.right.get assert(node.label == "properties") - val children = node.child.collect{case elem if elem.isElement => elem} + val children = node.children.collect{case elem if elem.isElement => elem} val props0 = children.toList.traverseU(Pom.property) assert(props0.isRight) diff --git a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala index fd9d0ea4d..e8dab1420 100644 --- a/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala +++ b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala @@ -2,6 +2,7 @@ package coursier package test import coursier.core.Repository +import coursier.maven.MavenRepository import utest._ import scala.async.Async.{ async, await } @@ -16,7 +17,7 @@ object ResolutionTests extends TestSuite { ) = Resolution(deps, filter = filter, forceVersions = forceVersions) .process - .run(Fetch.default(repositories)) + .run(Platform.fetch(repositories)) .runF implicit class ProjectOps(val p: Project) extends AnyVal { @@ -27,69 +28,68 @@ object ResolutionTests extends TestSuite { Project(Module("acme", "config"), "1.3.0"), Project(Module("acme", "play"), "2.4.0", Seq( - Dependency(Module("acme", "play-json"), "2.4.0"))), + "" -> Dependency(Module("acme", "play-json"), "2.4.0"))), Project(Module("acme", "play-json"), "2.4.0"), Project(Module("acme", "play"), "2.4.1", dependencies = Seq( - Dependency(Module("acme", "play-json"), "${playJsonVersion}"), - Dependency(Module("${project.groupId}", "${configName}"), "1.3.0")), - properties = Map( + "" -> Dependency(Module("acme", "play-json"), "${playJsonVersion}"), + "" -> Dependency(Module("${project.groupId}", "${configName}"), "1.3.0")), + properties = Seq( "playJsonVersion" -> "2.4.0", "configName" -> "config")), Project(Module("acme", "play-extra-no-config"), "2.4.1", Seq( - Dependency(Module("acme", "play"), "2.4.1", + "" -> Dependency(Module("acme", "play"), "2.4.1", exclusions = Set(("acme", "config"))))), Project(Module("acme", "play-extra-no-config-no"), "2.4.1", Seq( - Dependency(Module("acme", "play"), "2.4.1", + "" -> Dependency(Module("acme", "play"), "2.4.1", exclusions = Set(("*", "config"))))), Project(Module("hudsucker", "mail"), "10.0", Seq( - Dependency(Module("${project.groupId}", "test-util"), "${project.version}", - scope = Scope.Test))), + "test" -> Dependency(Module("${project.groupId}", "test-util"), "${project.version}"))), Project(Module("hudsucker", "test-util"), "10.0"), Project(Module("se.ikea", "parent"), "18.0", dependencyManagement = Seq( - Dependency(Module("acme", "play"), "2.4.0", + "" -> Dependency(Module("acme", "play"), "2.4.0", exclusions = Set(("acme", "play-json"))))), Project(Module("se.ikea", "billy"), "18.0", dependencies = Seq( - Dependency(Module("acme", "play"), "")), + "" -> Dependency(Module("acme", "play"), "")), parent = Some(Module("se.ikea", "parent"), "18.0")), Project(Module("org.gnome", "parent"), "7.0", Seq( - Dependency(Module("org.gnu", "glib"), "13.4"))), + "" -> Dependency(Module("org.gnu", "glib"), "13.4"))), Project(Module("org.gnome", "panel-legacy"), "7.0", dependencies = Seq( - Dependency(Module("org.gnome", "desktop"), "${project.version}")), + "" -> Dependency(Module("org.gnome", "desktop"), "${project.version}")), parent = Some(Module("org.gnome", "parent"), "7.0")), Project(Module("gov.nsa", "secure-pgp"), "10.0", Seq( - Dependency(Module("gov.nsa", "crypto"), "536.89"))), + "" -> Dependency(Module("gov.nsa", "crypto"), "536.89"))), Project(Module("com.mailapp", "mail-client"), "2.1", dependencies = Seq( - Dependency(Module("gov.nsa", "secure-pgp"), "10.0", + "" -> Dependency(Module("gov.nsa", "secure-pgp"), "10.0", exclusions = Set(("*", "${crypto.name}")))), - properties = Map("crypto.name" -> "crypto", "dummy" -> "2")), + properties = Seq("crypto.name" -> "crypto", "dummy" -> "2")), Project(Module("com.thoughtworks.paranamer", "paranamer-parent"), "2.6", dependencies = Seq( - Dependency(Module("junit", "junit"), "")), + "" -> Dependency(Module("junit", "junit"), "")), dependencyManagement = Seq( - Dependency(Module("junit", "junit"), "4.11", scope = Scope.Test))), + "test" -> Dependency(Module("junit", "junit"), "4.11"))), Project(Module("com.thoughtworks.paranamer", "paranamer"), "2.6", parent = Some(Module("com.thoughtworks.paranamer", "paranamer-parent"), "2.6")), @@ -97,75 +97,75 @@ object ResolutionTests extends TestSuite { Project(Module("com.github.dummy", "libb"), "0.3.3", profiles = Seq( Profile("default", activeByDefault = Some(true), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.4.2", dependencies = Seq( - Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4")), + "" -> Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4")), profiles = Seq( Profile("default", activeByDefault = Some(true), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"), - Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4", scope = Scope.Test))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"), + "test" -> Dependency(Module("org.scalaverification", "scala-verification"), "1.12.4"))))), Project(Module("com.github.dummy", "libb"), "0.5.3", - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> None)), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.5.4", - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("true"))), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb"), "0.5.5", - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("com.github.dummy", "libb-parent"), "0.5.6", - properties = Map("special" -> "true")), + properties = Seq("special" -> "true")), Project(Module("com.github.dummy", "libb"), "0.5.6", parent = Some(Module("com.github.dummy", "libb-parent"), "0.5.6"), - properties = Map("special" -> "true"), + properties = Seq("special" -> "true"), profiles = Seq( Profile("default", activation = Profile.Activation(properties = Seq("special" -> Some("!false"))), dependencies = Seq( - Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), + "" -> Dependency(Module("org.escalier", "librairie-standard"), "2.11.6"))))), Project(Module("an-org", "a-name"), "1.0"), Project(Module("an-org", "a-name"), "1.2"), Project(Module("an-org", "a-lib"), "1.0", - Seq(Dependency(Module("an-org", "a-name"), "1.0"))), + Seq("" -> Dependency(Module("an-org", "a-name"), "1.0"))), Project(Module("an-org", "a-lib"), "1.1"), Project(Module("an-org", "a-lib"), "1.2", - Seq(Dependency(Module("an-org", "a-name"), "1.2"))), + Seq("" -> Dependency(Module("an-org", "a-name"), "1.2"))), Project(Module("an-org", "another-lib"), "1.0", - Seq(Dependency(Module("an-org", "a-name"), "1.0"))), + Seq("" -> Dependency(Module("an-org", "a-name"), "1.0"))), // Must bring transitively an-org:a-name, as an optional dependency Project(Module("an-org", "an-app"), "1.0", Seq( - Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))), - Dependency(Module("an-org", "another-lib"), "1.0", optional = true))), + "" -> Dependency(Module("an-org", "a-lib"), "1.0", exclusions = Set(("an-org", "a-name"))), + "" -> Dependency(Module("an-org", "another-lib"), "1.0", optional = true))), Project(Module("an-org", "an-app"), "1.1", Seq( - Dependency(Module("an-org", "a-lib"), "1.1"))), + "" -> Dependency(Module("an-org", "a-lib"), "1.1"))), Project(Module("an-org", "an-app"), "1.2", Seq( - Dependency(Module("an-org", "a-lib"), "1.2"))) + "" -> Dependency(Module("an-org", "a-lib"), "1.2"))) ) - val projectsMap = projects.map(p => p.moduleVersion -> p).toMap + val projectsMap = projects.map(p => p.moduleVersion -> p.copy(configurations = MavenRepository.defaultConfigurations)).toMap val testRepository = new TestRepository(projectsMap) val repositories = Seq[Repository]( @@ -243,14 +243,11 @@ object ResolutionTests extends TestSuite { ) val res = await(resolve0( Set(dep) - )) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) + dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) assert(res == expected) @@ -267,14 +264,11 @@ object ResolutionTests extends TestSuite { ) val res = await(resolve0( Set(dep) - )) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) + dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) assert(res == expected) @@ -291,14 +285,11 @@ object ResolutionTests extends TestSuite { ) val res = await(resolve0( Set(dep) - )) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) ++ trDeps.map(trDep => projectsMap(trDep.moduleVersion).kv) + dependencies = Set(dep.withCompileScope) ++ trDeps.map(_.withCompileScope) ) assert(res == expected) @@ -308,16 +299,12 @@ object ResolutionTests extends TestSuite { async { val dep = Dependency(Module("hudsucker", "mail"), "10.0") val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) - )).copy(filter = None) + Set(dep) + )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( rootDependencies = Set(dep), - dependencies = Set(dep.withCompileScope), - projectCache = Map( - projectsMap(dep.moduleVersion).kv - ) + dependencies = Set(dep.withCompileScope) ) assert(res == expected) @@ -331,8 +318,7 @@ object ResolutionTests extends TestSuite { exclusions = Set(("acme", "play-json"))) ) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty) val expected = Resolution( @@ -350,8 +336,7 @@ object ResolutionTests extends TestSuite { Dependency(Module("org.gnu", "glib"), "13.4"), Dependency(Module("org.gnome", "desktop"), "7.0")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -368,8 +353,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("gov.nsa", "secure-pgp"), "10.0", exclusions = Set(("*", "crypto")))) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -384,8 +368,7 @@ object ResolutionTests extends TestSuite { async { val dep = Dependency(Module("com.thoughtworks.paranamer", "paranamer"), "2.6") val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -402,8 +385,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -422,8 +404,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -444,8 +425,7 @@ object ResolutionTests extends TestSuite { val trDeps = Seq( Dependency(Module("org.escalier", "librairie-standard"), "2.11.6")) val res = await(resolve0( - Set(dep), - filter = Some(_.scope == Scope.Compile) + Set(dep) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -466,7 +446,7 @@ object ResolutionTests extends TestSuite { Dependency(Module("an-org", "a-name"), "1.0", optional = true)) val res = await(resolve0( Set(dep), - filter = Some(_.scope == Scope.Compile) + filter = Some(_ => true) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -489,7 +469,7 @@ object ResolutionTests extends TestSuite { Dependency(Module("an-org", "a-name"), "1.0", optional = true)) val res = await(resolve0( deps, - filter = Some(_.scope == Scope.Compile) + filter = Some(_ => true) )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -511,8 +491,7 @@ object ResolutionTests extends TestSuite { val res = await(resolve0( deps, - forceVersions = depOverrides, - filter = Some(_.scope == Scope.Compile) + forceVersions = depOverrides )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -536,8 +515,7 @@ object ResolutionTests extends TestSuite { val res = await(resolve0( deps, - forceVersions = depOverrides, - filter = Some(_.scope == Scope.Compile) + forceVersions = depOverrides )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -563,8 +541,7 @@ object ResolutionTests extends TestSuite { val res = await(resolve0( deps, - forceVersions = depOverrides, - filter = Some(_.scope == Scope.Compile) + forceVersions = depOverrides )).copy(filter = None, projectCache = Map.empty, errorCache = Map.empty) val expected = Resolution( @@ -585,9 +562,9 @@ object ResolutionTests extends TestSuite { 'propertySubstitution{ val res = core.Resolution.withProperties( - Seq(Dependency(Module("a-company", "a-name"), "${a.property}")), + Seq("" -> Dependency(Module("a-company", "a-name"), "${a.property}")), Map("a.property" -> "a-version")) - val expected = Seq(Dependency(Module("a-company", "a-name"), "a-version")) + val expected = Seq("" -> Dependency(Module("a-company", "a-name"), "a-version")) assert(res == expected) } diff --git a/tests/shared/src/test/scala/coursier/test/TestRepository.scala b/tests/shared/src/test/scala/coursier/test/TestRepository.scala index 85144d66c..bc851b8e1 100644 --- a/tests/shared/src/test/scala/coursier/test/TestRepository.scala +++ b/tests/shared/src/test/scala/coursier/test/TestRepository.scala @@ -8,12 +8,16 @@ import scalaz.Scalaz._ class TestRepository(projects: Map[(Module, String), Project]) extends Repository { val source = new core.Artifact.Source { - def artifacts(dependency: Dependency, project: Project) = ??? + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ) = ??? } def find[F[_]]( module: Module, version: String, - fetch: Repository.Fetch[F] + fetch: Fetch.Content[F] )(implicit F: Monad[F] ) = diff --git a/tests/shared/src/test/scala/coursier/test/package.scala b/tests/shared/src/test/scala/coursier/test/package.scala index ebcf0b6ab..30fa8f3ea 100644 --- a/tests/shared/src/test/scala/coursier/test/package.scala +++ b/tests/shared/src/test/scala/coursier/test/package.scala @@ -3,7 +3,7 @@ package coursier package object test { implicit class DependencyOps(val underlying: Dependency) extends AnyVal { - def withCompileScope: Dependency = underlying.copy(scope = Scope.Compile) + def withCompileScope: Dependency = underlying.copy(configuration = "compile") } object Profile { @@ -13,25 +13,51 @@ package object test { core.Activation(properties) } - def apply(id: String, - activeByDefault: Option[Boolean] = None, - activation: Activation = Activation(), - dependencies: Seq[Dependency] = Nil, - dependencyManagement: Seq[Dependency] = Nil, - properties: Map[String, String] = Map.empty) = - core.Profile(id, activeByDefault, activation, dependencies, dependencyManagement, properties) + def apply( + id: String, + activeByDefault: Option[Boolean] = None, + activation: Activation = Activation(), + dependencies: Seq[(String, Dependency)] = Nil, + dependencyManagement: Seq[(String, Dependency)] = Nil, + properties: Map[String, String] = Map.empty + ) = + core.Profile( + id, + activeByDefault, + activation, + dependencies, + dependencyManagement, + properties + ) } object Project { - def apply(module: Module, - version: String, - dependencies: Seq[Dependency] = Seq.empty, - parent: Option[ModuleVersion] = None, - dependencyManagement: Seq[Dependency] = Seq.empty, - properties: Map[String, String] = Map.empty, - profiles: Seq[Profile] = Seq.empty, - versions: Option[core.Versions] = None, - snapshotVersioning: Option[core.SnapshotVersioning] = None): Project = - core.Project(module, version, dependencies, parent, dependencyManagement, properties, profiles, versions, snapshotVersioning) + def apply( + module: Module, + version: String, + dependencies: Seq[(String, Dependency)] = Seq.empty, + parent: Option[ModuleVersion] = None, + dependencyManagement: Seq[(String, Dependency)] = Seq.empty, + configurations: Map[String, Seq[String]] = Map.empty, + properties: Seq[(String, String)] = Seq.empty, + profiles: Seq[Profile] = Seq.empty, + versions: Option[core.Versions] = None, + snapshotVersioning: Option[core.SnapshotVersioning] = None, + publications: Seq[(String, core.Publication)] = Nil + ): Project = + core.Project( + module, + version, + dependencies, + configurations, + parent, + dependencyManagement, + properties, + profiles, + versions, + snapshotVersioning, + publications, + Info.empty + ) } } diff --git a/version.sbt b/version.sbt index aab7f9b81..6dc058896 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.1.0-SNAPSHOT" \ No newline at end of file +version in ThisBuild := "1.0.0-SNAPSHOT" diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 56117eb1c..69e6c3d96 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -11,14 +11,12 @@ import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue import org.scalajs.jquery.jQuery import scala.concurrent.Future -import scalaz.concurrent.Task import scala.scalajs.js import js.Dynamic.{ global => g } case class ResolutionOptions( - followOptional: Boolean = false, - keepTest: Boolean = false + followOptional: Boolean = false ) case class State( @@ -37,12 +35,12 @@ class Backend($: BackendScope[Unit, State]) { def fetch( repositories: Seq[core.Repository], - fetch: Repository.Fetch[Task] - ): ResolutionProcess.Fetch[Task] = { + fetch: Fetch.Content[Task] + ): Fetch.Metadata[Task] = { modVers => Task.gatherUnordered( modVers.map { case (module, version) => - Repository.find(repositories, module, version, fetch) + Fetch.find(repositories, module, version, fetch) .run .map((module, version) -> _) } @@ -66,7 +64,7 @@ class Backend($: BackendScope[Unit, State]) { Seq( dep.module.organization, dep.module.name, - dep.scope.name + dep.configuration ).mkString(":") for { @@ -176,8 +174,7 @@ class Backend($: BackendScope[Unit, State]) { val res = coursier.Resolution( s.modules.toSet, filter = Some(dep => - (s.options.followOptional || !dep.optional) && - (s.options.keepTest || dep.scope != Scope.Test) + s.options.followOptional || !dep.optional ) ) @@ -338,14 +335,6 @@ class Backend($: BackendScope[Unit, State]) { ) ) } - def toggleTest(e: ReactEventI) = { - $.modState(s => - s.copy( - options = s.options - .copy(keepTest = !s.options.keepTest) - ) - ) - } } } @@ -380,7 +369,7 @@ object App { <.td(dep.module.name), <.td(finalVersionOpt.fold(dep.version)(finalVersion => s"$finalVersion (for ${dep.version})")), <.td(Seq[Seq[TagMod]]( - if (dep.scope == Scope.Compile) Seq() else Seq(infoLabel(dep.scope.name)), + if (dep.configuration == "compile") Seq() else Seq(infoLabel(dep.configuration)), if (dep.attributes.`type`.isEmpty || dep.attributes.`type` == "jar") Seq() else Seq(infoLabel(dep.attributes.`type`)), if (dep.attributes.classifier.isEmpty) Seq() else Seq(infoLabel(dep.attributes.classifier)), Some(dep.exclusions).filter(_.nonEmpty).map(excls => infoPopOver("Exclusions", excls.toList.sorted.map{case (org, name) => s"$org:$name"}.mkString("; "))).toSeq, @@ -417,7 +406,10 @@ object App { } val sortedDeps = res.minDependencies.toList - .sortBy(dep => coursier.core.Module.unapply(dep.module).get) + .sortBy { dep => + val (org, name, _) = coursier.core.Module.unapply(dep.module).get + (org, name) + } <.table(^.`class` := "table", <.thead( @@ -683,15 +675,6 @@ object App { "Follow optional dependencies" ) ) - ), - <.div(^.`class` := "checkbox", - <.label( - <.input(^.`type` := "checkbox", - ^.onChange ==> backend.options.toggleTest, - if (options.keepTest) Seq(^.checked := "checked") else Seq(), - "Keep test dependencies" - ) - ) ) ) }