diff --git a/appveyor.yml b/appveyor.yml index a5ef6d67e..a575e0913 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,7 +16,7 @@ install: build_script: - sbt clean compile publish-local test_script: - - sbt coreJVM/test # Would node be around for coreJS/test? + - sbt testsJVM/test # Would node be around for testsJS/test? cache: - C:\sbt\ - C:\Users\appveyor\.m2 diff --git a/build.sbt b/build.sbt index 42f5c9f3f..884e2a538 100644 --- a/build.sbt +++ b/build.sbt @@ -65,41 +65,65 @@ lazy val commonSettings = baseCommonSettings ++ Seq( } ) + lazy val core = crossProject .settings(commonSettings: _*) .settings(publishingSettings: _*) .settings( - name := "coursier", - libraryDependencies += "org.scala-lang.modules" %% "scala-async" % "0.9.1" % "provided", - unmanagedResourceDirectories in Compile += (baseDirectory in LocalRootProject).value / "core" / "shared" / "src" / "main" / "resources", - unmanagedResourceDirectories in Test += (baseDirectory in LocalRootProject).value / "core" / "shared" / "src" / "test" / "resources", - testFrameworks += new TestFramework("utest.runner.Framework") + name := "coursier" ) .jvmSettings( - libraryDependencies ++= Seq( - "org.scalaz" %% "scalaz-concurrent" % "7.1.2", - "com.lihaoyi" %% "utest" % "0.3.0" % "test" - ) ++ { - if (scalaVersion.value.startsWith("2.10.")) Seq() - else Seq( - "org.scala-lang.modules" %% "scala-xml" % "1.0.3" - ) - } + libraryDependencies ++= + Seq( + "org.scalaz" %% "scalaz-core" % "7.1.2" + ) ++ { + if (scalaVersion.value.startsWith("2.10.")) Seq() + else Seq( + "org.scala-lang.modules" %% "scala-xml" % "1.0.3" + ) + } ) .jsSettings( libraryDependencies ++= Seq( - "org.scala-js" %%% "scalajs-dom" % "0.8.0", "com.github.japgolly.fork.scalaz" %%% "scalaz-core" % (if (scalaVersion.value.startsWith("2.10.")) "7.1.1" else "7.1.2"), - "be.doeraene" %%% "scalajs-jquery" % "0.8.0", - "com.lihaoyi" %%% "utest" % "0.3.0" % "test" - ), - postLinkJSEnv := NodeJSEnv().value, - scalaJSStage in Global := FastOptStage + "org.scala-js" %%% "scalajs-dom" % "0.8.0", + "be.doeraene" %%% "scalajs-jquery" % "0.8.0" + ) ) lazy val coreJvm = core.jvm lazy val coreJs = core.js +lazy val `fetch-js` = project + .enablePlugins(ScalaJSPlugin) + .dependsOn(coreJs) + .settings(commonSettings) + .settings(noPublishSettings) + .settings( + name := "coursier-fetch-js" + ) + +lazy val tests = crossProject + .dependsOn(core) + .settings(commonSettings: _*) + .settings(noPublishSettings: _*) + .settings( + name := "coursier-tests", + libraryDependencies ++= Seq( + "org.scala-lang.modules" %% "scala-async" % "0.9.1" % "provided", + "com.lihaoyi" %%% "utest" % "0.3.0" % "test" + ), + unmanagedResourceDirectories in Test += (baseDirectory in LocalRootProject).value / "tests" / "shared" / "src" / "test" / "resources", + testFrameworks += new TestFramework("utest.runner.Framework") + ) + .jsSettings( + postLinkJSEnv := NodeJSEnv().value, + scalaJSStage in Global := FastOptStage + ) + +lazy val testsJvm = tests.jvm.dependsOn(files % "test") +lazy val testsJs = tests.js.dependsOn(`fetch-js` % "test") + lazy val files = project .dependsOn(coreJvm) .settings(commonSettings) @@ -107,25 +131,38 @@ lazy val files = project .settings( name := "coursier-files", libraryDependencies ++= Seq( - "com.lihaoyi" %% "utest" % "0.3.0" % "test" - ), - testFrameworks += new TestFramework("utest.runner.Framework") + "org.scalaz" %% "scalaz-concurrent" % "7.1.2" + ) + ) + +lazy val bootstrap = project + .settings(baseCommonSettings) + .settings(publishingSettings) + .settings( + name := "coursier-bootstrap", + artifactName := { + val artifactName0 = artifactName.value + (sv, m, artifact) => + if (artifact.`type` == "jar" && artifact.extension == "jar") + "bootstrap.jar" + else + artifactName0(sv, m, artifact) + }, + crossPaths := false, + autoScalaLibrary := false, + javacOptions in doc := Seq() ) lazy val cli = project .dependsOn(coreJvm, files) .settings(commonSettings) .settings(publishingSettings) - .settings(packAutoSettings ++ publishPackTxzArchive ++ publishPackZipArchive) - .settings( - packArchivePrefix := s"coursier-cli_${scalaBinaryVersion.value}", - packArchiveTxzArtifact := Artifact("coursier-cli", "arch", "tar.xz"), - packArchiveZipArtifact := Artifact("coursier-cli", "arch", "zip") - ) + .settings(packAutoSettings) .settings( name := "coursier-cli", libraryDependencies ++= Seq( "com.github.alexarchambault" %% "case-app" % "1.0.0-SNAPSHOT", + "com.lihaoyi" %% "ammonite-terminal" % "0.5.0", "ch.qos.logback" % "logback-classic" % "1.1.3" ), resourceGenerators in Compile += packageBin.in(bootstrap).in(Compile).map { jar => @@ -135,7 +172,7 @@ lazy val cli = project lazy val web = project .enablePlugins(ScalaJSPlugin) - .dependsOn(coreJs) + .dependsOn(coreJs, `fetch-js`) .settings(commonSettings) .settings(noPublishSettings) .settings( @@ -164,29 +201,7 @@ lazy val web = project ) ) -lazy val bootstrap = project - .settings(baseCommonSettings) - .settings(publishingSettings) - .settings( - name := "coursier-bootstrap", - artifactName := { - val artifactName0 = artifactName.value - (sv, m, artifact) => - if (artifact.`type` == "jar" && artifact.extension == "jar") - "bootstrap.jar" - else - artifactName0(sv, m, artifact) - }, - crossPaths := false, - autoScalaLibrary := false, - javacOptions in doc := Seq() - ) - lazy val `coursier` = project.in(file(".")) - .aggregate(coreJvm, coreJs, files, cli, web, bootstrap) + .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web) .settings(commonSettings) .settings(noPublishSettings) - .settings( - (unmanagedSourceDirectories in Compile) := Nil, - (unmanagedSourceDirectories in Test) := Nil - ) diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index b1ea1710d..190e87442 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -36,7 +36,7 @@ case class CommonOptions( @Recurse cacheOptions: CacheOptions ) { - val verbose0 = verbose.length + (if (quiet) 1 else 0) + val verbose0 = verbose.length - (if (quiet) 1 else 0) } object CacheOptions { @@ -70,7 +70,7 @@ case class Fetch( val files0 = helper.fetch(main = true, sources = false, javadoc = false) - Console.out.println( + println( files0 .map(_.toString) .mkString("\n") @@ -115,7 +115,7 @@ case class Launch( val mainClass = if (mainClasses.isEmpty) { - Console.err.println(s"No main class found. Specify one with -M or --main.") + Helper.errPrintln("No main class found. Specify one with -M or --main.") sys.exit(255) } else if (mainClasses.size == 1) { val (_, mainClass) = mainClasses.head @@ -135,8 +135,7 @@ case class Launch( } yield mainClass mainClassOpt.getOrElse { - println(mainClasses) - Console.err.println(s"Cannot find default main class. Specify one with -M or --main.") + Helper.errPrintln(s"Cannot find default main class. Specify one with -M or --main.") sys.exit(255) } } @@ -147,18 +146,20 @@ case class Launch( val cls = try cl.loadClass(mainClass0) catch { case e: ClassNotFoundException => - println(s"Error: class $mainClass0 not found") + Helper.errPrintln(s"Error: class $mainClass0 not found") sys.exit(255) } val method = try cls.getMethod("main", classOf[Array[String]]) catch { case e: NoSuchMethodError => - println(s"Error: method main not found in $mainClass0") + Helper.errPrintln(s"Error: method main not found in $mainClass0") sys.exit(255) } if (common.verbose0 >= 1) - println(s"Calling $mainClass0 ${extraArgs.mkString(" ")}") + Helper.errPrintln(s"Launching $mainClass0 ${extraArgs.mkString(" ")}") + else if (common.verbose0 == 0) + Helper.errPrintln(s"Launching") Thread.currentThread().setContextClassLoader(cl) method.invoke(null, extraArgs.toArray) @@ -326,13 +327,7 @@ case class Bootstrap( sys.exit(1) } - // scala-library version in the resulting JARs has to match the one in the bootstrap JAR - // This should be enforced more strictly (possibly by having one bootstrap JAR per scala version). - - val helper = new Helper( - common, - remainingArgs :+ s"org.scala-lang:scala-library:${scala.util.Properties.versionNumberString}" - ) + val helper = new Helper(common, remainingArgs) val artifacts = helper.res.artifacts diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index c46df1c1e..8f303cc25 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -1,12 +1,9 @@ -package coursier.cli +package coursier +package cli -import java.io.File +import java.io.{ OutputStreamWriter, File } import java.util.UUID -import caseapp.CaseApp -import coursier._ -import coursier.core.{ CachePolicy, MavenRepository } - import scalaz.{ \/-, -\/ } import scalaz.concurrent.Task @@ -31,50 +28,6 @@ object Helper { def errPrintln(s: String) = Console.err.println(s) - def defaultLogger: MavenRepository.Logger with Files.Logger = - new MavenRepository.Logger with Files.Logger { - def downloading(url: String) = - errPrintln(s"Downloading $url") - def downloaded(url: String, success: Boolean) = - if (!success) - errPrintln(s"Failed: $url") - def readingFromCache(f: File) = {} - def puttingInCache(f: File) = {} - - def foundLocally(f: File) = {} - def downloadingArtifact(url: String) = - errPrintln(s"Downloading $url") - def downloadedArtifact(url: String, success: Boolean) = - if (!success) - errPrintln(s"Failed: $url") - } - - def verboseLogger: MavenRepository.Logger with Files.Logger = - new MavenRepository.Logger with Files.Logger { - def downloading(url: String) = - errPrintln(s"Downloading $url") - def downloaded(url: String, success: Boolean) = - errPrintln( - if (success) s"Downloaded $url" - else s"Failed: $url" - ) - def readingFromCache(f: File) = { - errPrintln(s"Reading ${fileRepr(f)} from cache") - } - def puttingInCache(f: File) = - errPrintln(s"Writing ${fileRepr(f)} in cache") - - def foundLocally(f: File) = - errPrintln(s"Found locally ${fileRepr(f)}") - def downloadingArtifact(url: String) = - errPrintln(s"Downloading $url") - def downloadedArtifact(url: String, success: Boolean) = - errPrintln( - if (success) s"Downloaded $url" - else s"Failed: $url" - ) - } - def mainClasses(cl: ClassLoader): Map[(String, String), String] = { import scala.collection.JavaConverters._ @@ -101,15 +54,6 @@ class Helper( import common._ import Helper.errPrintln - - val logger = - if (verbose0 < 0) - None - else if (verbose0 == 0) - Some(Helper.defaultLogger) - else - Some(Helper.verboseLogger) - implicit val cachePolicy = if (offline) CachePolicy.LocalOnly @@ -176,13 +120,12 @@ class Helper( sys.exit(1) } - val (repositories0, fileCaches) = repositoryIdsOpt0 + val files = cache.files().copy(concurrentDownloadCount = parallel) + + val (repositories, fileCaches) = repositoryIdsOpt0 .collect { case Right(v) => v } .unzip - val repositories = repositories0 - .map(_.copy(logger = logger)) - val (rawDependencies, extraArgs) = { val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0) idxOpt.fold((remainingArgs, Seq.empty[String])) { idx => @@ -221,9 +164,15 @@ class Helper( filter = Some(dep => keepOptional || !dep.optional) ) - val fetchQuiet = coursier.fetchLocalFirst(repositories) + val logger = + if (verbose0 >= 0) + Some(new TermDisplay(new OutputStreamWriter(System.err))) + else + None + logger.foreach(_.init()) + val fetchQuiet = coursier.Fetch(repositories, files.fetch(logger = logger)) val fetch0 = - if (verbose0 == 0) fetchQuiet + if (verbose0 <= 0) fetchQuiet else { modVers: Seq[(Module, String)] => val print = Task{ @@ -241,6 +190,8 @@ class Helper( .run(fetch0, maxIterations) .run + logger.foreach(_.stop()) + if (!res.isDone) { errPrintln(s"Maximum number of iteration reached!") sys.exit(1) @@ -277,7 +228,7 @@ class Helper( .toList .sortBy(repr) - if (verbose0 >= 0) { + if (verbose0 >= 1) { println("") println( trDeps @@ -301,8 +252,8 @@ class Helper( } def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = { - println("") - + if (verbose0 >= 0) + errPrintln("Fetching artifacts") val artifacts0 = res.artifacts val main0 = main || (!sources && !javadoc) val artifacts = artifacts0.flatMap{ artifact => @@ -317,17 +268,15 @@ class Helper( l } - val files = { - var files0 = cache - .files() - .copy(logger = logger) - files0 = files0.copy(concurrentDownloadCount = parallel) - files0 - } - - val tasks = artifacts.map(artifact => files.file(artifact).run.map(artifact.->)) - def printTask = Task{ - if (verbose0 >= 0 && artifacts.nonEmpty) + val logger = + if (verbose0 >= 0) + Some(new TermDisplay(new OutputStreamWriter(System.err))) + else + None + logger.foreach(_.init()) + val tasks = artifacts.map(artifact => files.file(artifact, logger = logger).run.map(artifact.->)) + def printTask = Task { + if (verbose0 >= 1 && artifacts.nonEmpty) println(s"Found ${artifacts.length} artifacts") } val task = printTask.flatMap(_ => Task.gatherUnordered(tasks)) @@ -336,6 +285,8 @@ class Helper( val errors = results.collect{case (artifact, -\/(err)) => artifact -> err } val files0 = results.collect{case (artifact, \/-(f)) => f } + logger.foreach(_.stop()) + if (errors.nonEmpty) { println(s"${errors.size} error(s):") for ((artifact, error) <- errors) { diff --git a/cli/src/main/scala/coursier/cli/TermDisplay.scala b/cli/src/main/scala/coursier/cli/TermDisplay.scala new file mode 100644 index 000000000..acdcbaf52 --- /dev/null +++ b/cli/src/main/scala/coursier/cli/TermDisplay.scala @@ -0,0 +1,146 @@ +package coursier.cli + +import java.io.Writer +import java.util.concurrent._ + +import ammonite.terminal.{ TTY, Ansi } + +import coursier.Files.Logger + +import scala.annotation.tailrec +import scala.collection.mutable.ArrayBuffer + +class TermDisplay(out: Writer) extends Logger { + + private val ansi = new Ansi(out) + private var width = 80 + private val refreshInterval = 1000 / 60 + private val lock = new AnyRef + private val t = new Thread("TermDisplay") { + override def run() = lock.synchronized { + val baseExtraWidth = width / 5 + @tailrec def helper(lineCount: Int): Unit = + Option(q.poll(100L, TimeUnit.MILLISECONDS)) match { + case None => helper(lineCount) + case Some(Left(())) => // poison pill + case Some(Right(())) => + // update display + + for (_ <- 0 until lineCount) { + ansi.up(1) + ansi.clearLine(2) + } + + val downloads0 = downloads.synchronized { + downloads + .toVector + .map { url => url -> infos.get(url) } + .sortBy { case (_, info) => - info.pct.sum } + } + + for ((url, info) <- downloads0) { + assert(info != null, s"Incoherent state ($url)") + val pctOpt = info.pct.map(100.0 * _) + val extra = s"(${pctOpt.map(pct => f"$pct%.2f %%, ").mkString}${info.downloaded}${info.length.map(" / " + _).mkString})" + + val total = url.length + 1 + extra.length + val (url0, extra0) = + if (total >= width) { // or > ? If equal, does it go down 2 lines? + val overflow = total - width + 1 + + val extra0 = + if (extra.length > baseExtraWidth) + extra.take((baseExtraWidth max (extra.length - overflow)) - 1) + "…" + else + extra + + val total0 = url.length + 1 + extra0.length + val overflow0 = total0 - width + 1 + + val url0 = + if (total0 >= width) + url.take(((width - baseExtraWidth - 1) max (url.length - overflow0)) - 1) + "…" + else + url + + (url0, extra0) + } else + (url, extra) + + out.write(s"$url0 $extra0\n") + } + + out.flush() + Thread.sleep(refreshInterval) + helper(downloads0.length) + } + + helper(0) + } + } + + t.setDaemon(true) + + def init(): Unit = { + width = TTY.consoleDim("cols") + ansi.clearLine(2) + t.start() + } + + def stop(): Unit = { + q.put(Left(())) + lock.synchronized(()) + } + + private case class Info(downloaded: Long, length: Option[Long]) { + def pct: Option[Double] = length.map(downloaded.toDouble / _) + } + + private val downloads = new ArrayBuffer[String] + private val infos = new ConcurrentHashMap[String, Info] + + private val q = new LinkedBlockingDeque[Either[Unit, Unit]] + def update(): Unit = { + if (q.size() == 0) + q.put(Right(())) + } + + override def downloadingArtifact(url: String): Unit = { + assert(!infos.containsKey(url)) + val prev = infos.putIfAbsent(url, Info(0L, None)) + assert(prev == null) + + downloads.synchronized { + downloads.append(url) + } + + update() + } + override def downloadLength(url: String, length: Long): Unit = { + val info = infos.get(url) + assert(info != null) + val newInfo = info.copy(length = Some(length)) + infos.put(url, newInfo) + + update() + } + override def downloadProgress(url: String, downloaded: Long): Unit = { + val info = infos.get(url) + assert(info != null) + val newInfo = info.copy(downloaded = downloaded) + infos.put(url, newInfo) + + update() + } + override def downloadedArtifact(url: String, success: Boolean): Unit = { + downloads.synchronized { + downloads -= url + } + + val info = infos.remove(url) + assert(info != null) + + update() + } + +} diff --git a/core/js/src/main/scala/coursier/core/compatibility/package.scala b/core/js/src/main/scala/coursier/core/compatibility/package.scala index 34c0f71de..9226f6d07 100644 --- a/core/js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/js/src/main/scala/coursier/core/compatibility/package.scala @@ -4,6 +4,8 @@ import scala.scalajs.js import js.Dynamic.{ global => g } import org.scalajs.dom.raw.NodeList +import coursier.util.Xml + package object compatibility { def option[A](a: js.Dynamic): Option[A] = if (js.isUndefined(a)) None diff --git a/core/jvm/src/main/scala/coursier/core/MavenRepository.scala b/core/jvm/src/main/scala/coursier/core/MavenRepository.scala deleted file mode 100644 index c7665d1ed..000000000 --- a/core/jvm/src/main/scala/coursier/core/MavenRepository.scala +++ /dev/null @@ -1,135 +0,0 @@ -package coursier -package core - -import java.io._ -import java.net.{ URI, URL } - -import scala.io.Codec -import scalaz._, Scalaz._ -import scalaz.concurrent.Task - -case class MavenRepository( - root: String, - cache: Option[File] = None, - ivyLike: Boolean = false, - logger: Option[MavenRepository.Logger] = None -) extends BaseMavenRepository(root, ivyLike) { - - val isLocal = root.startsWith("file:/") - - def fetch( - artifact: Artifact, - cachePolicy: CachePolicy - ): EitherT[Task, String, String] = { - - def locally(eitherFile: String \/ File) = { - Task { - for { - f0 <- eitherFile - f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache") - content <- \/.fromTryCatchNonFatal{ - logger.foreach(_.readingFromCache(f)) - scala.io.Source.fromFile(f)(Codec.UTF8).mkString - }.leftMap(_.getMessage) - } yield content - } - } - - if (isLocal) EitherT(locally(\/-(new File(new URI(root + artifact.url) .getPath)))) - else { - lazy val localFile = { - for { - cache0 <- cache.toRightDisjunction("No cache") - f = new File(cache0, artifact.url) - } yield f - } - - def remote = { - val urlStr = root + artifact.url - val url = new URL(urlStr) - - def log = Task(logger.foreach(_.downloading(urlStr))) - def get = { - val conn = url.openConnection() - // Dummy user-agent instead of the default "Java/...", - // so that we are not returned incomplete/erroneous metadata - // (Maven 2 compatibility? - happens for snapshot versioning metadata, - // this is SO FUCKING CRAZY) - conn.setRequestProperty("User-Agent", "") - MavenRepository.readFully(conn.getInputStream()) - } - def logEnd(success: Boolean) = logger.foreach(_.downloaded(urlStr, success)) - - log - .flatMap(_ => get) - .map{ res => logEnd(res.isRight); res } - } - - def save(s: String) = { - localFile.fold(_ => Task.now(()), f => - Task { - if (!f.exists()) { - logger.foreach(_.puttingInCache(f)) - f.getParentFile.mkdirs() - val w = new PrintWriter(f) - try w.write(s) - finally w.close() - () - } - } - ) - } - - EitherT( - cachePolicy[String \/ String]( - _.isLeft )( - locally(localFile) )( - _ => CachePolicy.saving(remote)(save) - ) - ) - } - } - -} - -object MavenRepository { - - trait Logger { - def downloading(url: String): Unit - def downloaded(url: String, success: Boolean): Unit - def readingFromCache(f: File): Unit - def puttingInCache(f: File): Unit - } - - def readFullySync(is: InputStream) = { - val buffer = new ByteArrayOutputStream() - val data = Array.ofDim[Byte](16384) - - var nRead = is.read(data, 0, data.length) - while (nRead != -1) { - buffer.write(data, 0, nRead) - nRead = is.read(data, 0, data.length) - } - - buffer.flush() - buffer.toByteArray - } - - def readFully(is: => InputStream) = - Task { - \/.fromTryCatchNonFatal { - val is0 = is - val b = - try readFullySync(is0) - finally is0.close() - - new String(b, "UTF-8") - } .leftMap{ - case e: java.io.FileNotFoundException => - s"Not found: ${e.getMessage}" - case e => - s"$e: ${e.getMessage}" - } - } - -} diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index eb4cb154c..153b376e0 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -1,5 +1,7 @@ package coursier.core +import coursier.util.Xml + package object compatibility { implicit class RichChar(val c: Char) extends AnyVal { @@ -27,4 +29,5 @@ package object compatibility { def encodeURIComponent(s: String): String = new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString + } diff --git a/core/shared/src/main/scala/coursier/core/Repository.scala b/core/shared/src/main/scala/coursier/core/Repository.scala index e0e570920..69bcfa457 100644 --- a/core/shared/src/main/scala/coursier/core/Repository.scala +++ b/core/shared/src/main/scala/coursier/core/Repository.scala @@ -1,30 +1,24 @@ package coursier.core -import scalaz.{ -\/, \/-, \/, EitherT } -import scalaz.concurrent.Task +import scala.language.higherKinds -import java.io.File +import scalaz._ import coursier.core.compatibility.encodeURIComponent trait Repository { - def find( + def find[F[_]]( module: Module, - version: String + version: String, + fetch: Repository.Fetch[F] )(implicit - cachePolicy: CachePolicy - ): EitherT[Task, String, (Artifact.Source, Project)] + F: Monad[F] + ): EitherT[F, String, (Artifact.Source, Project)] } object Repository { - val mavenCentral = MavenRepository("https://repo1.maven.org/maven2/") - - val sonatypeReleases = MavenRepository("https://oss.sonatype.org/content/repositories/releases/") - val sonatypeSnapshots = MavenRepository("https://oss.sonatype.org/content/repositories/snapshots/") - - lazy val ivy2Local = MavenRepository(new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, ivyLike = true) - + type Fetch[F[_]] = Artifact => EitherT[F, String, String] /** * Try to find `module` among `repositories`. @@ -37,38 +31,33 @@ object Repository { * version (e.g. version interval). Which version get chosen depends on * the repository implementation. */ - def find( + def find[F[_]]( repositories: Seq[Repository], module: Module, - version: String + version: String, + fetch: Repository.Fetch[F] )(implicit - cachePolicy: CachePolicy - ): EitherT[Task, Seq[String], (Artifact.Source, Project)] = { + F: Monad[F] + ): EitherT[F, Seq[String], (Artifact.Source, Project)] = { val lookups = repositories - .map(repo => repo -> repo.find(module, version).run) + .map(repo => repo -> repo.find(module, version, fetch).run) - val task = lookups - .foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Artifact.Source, Project)]) { - case (acc, (repo, eitherProjTask)) => - acc - .flatMap { - case -\/(errors) => - eitherProjTask - .map(res => res - .flatMap{case (source, project) => - if (project.module == module) \/-((source, project)) - else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") - } - .leftMap(error => error +: errors) - ) + val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) { + case (acc, (repo, eitherProjTask)) => + F.bind(acc) { + case -\/(errors) => + F.map(eitherProjTask)(_.flatMap{case (source, project) => + if (project.module == module) \/-((source, project)) + else -\/(s"Wrong module returned (expected: $module, got: ${project.module})") + }.leftMap(error => error +: errors)) - case res @ \/-(_) => - Task.now(res) - } - } + case res @ \/-(_) => + F.point(res) + } + } - EitherT(task.map(_.leftMap(_.reverse))) + EitherT(F.map(task)(_.leftMap(_.reverse))) .map {case x @ (_, proj) => assert(proj.module == module) x @@ -101,401 +90,3 @@ object Repository { } } -case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { - import Repository._ - import BaseMavenRepository._ - - def artifacts( - dependency: Dependency, - project: Project - ): Seq[Artifact] = { - - def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = - ivyLikePath( - dependency.module.organization, - dependency.module.name, - project.version, - subDir, - baseSuffix, - ext - ) - - val path = - if (ivyLike) - ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) - else { - val versioning = - project - .snapshotVersioning - .flatMap(versioning => - mavenVersioning(versioning, dependency.attributes.classifier, dependency.attributes.`type`) - ) - - dependency.module.organization.split('.').toSeq ++ Seq( - dependency.module.name, - project.version, - s"${dependency.module.name}-${versioning getOrElse project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" - ) - } - - var artifact = - Artifact( - root + path.mkString("/"), - Map.empty, - Map.empty, - dependency.attributes - ) - .withDefaultChecksums - - if (dependency.attributes.`type` == "jar") { - artifact = artifact.withDefaultSignature - - // FIXME Snapshot versioning of sources and javadoc is not taken into account here. - // Will be ok if it's the same as the main JAR though. - - artifact = - if (ivyLike) { - val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") - val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") - - artifact - .copy( - extra = artifact.extra ++ Map( - "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? - .withDefaultChecksums - .withDefaultSignature, - "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above - .withDefaultChecksums - .withDefaultSignature - )) - } else - artifact - .withJavadocSources - } - - Seq(artifact) - } -} - -object BaseMavenRepository { - - def ivyLikePath( - org: String, - name: String, - version: String, - subDir: String, - baseSuffix: String, - ext: String - ) = - Seq( - org, - name, - version, - subDir, - s"$name$baseSuffix.$ext" - ) - - def mavenVersioning( - snapshotVersioning: SnapshotVersioning, - classifier: String, - extension: String - ): Option[String] = - snapshotVersioning - .snapshotVersions - .find(v => v.classifier == classifier && v.extension == extension) - .map(_.value) - .filter(_.nonEmpty) - -} - -abstract class BaseMavenRepository( - root: String, - ivyLike: Boolean -) extends Repository { - - def fetch( - artifact: Artifact, - cachePolicy: CachePolicy - ): EitherT[Task, String, String] - - import Repository._ - import BaseMavenRepository._ - - val source = MavenSource(root, ivyLike) - - def projectArtifact( - module: Module, - version: String, - versioningValue: Option[String] - ): Artifact = { - - val path = ( - if (ivyLike) - ivyLikePath( - module.organization, - module.name, - versioningValue getOrElse version, - "poms", - "", - "pom" - ) - else - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - s"${module.name}-${versioningValue getOrElse version}.pom" - ) - ) .map(encodeURIComponent) - - Artifact( - path.mkString("/"), - Map.empty, - Map.empty, - Attributes("pom", "") - ) - .withDefaultSignature - } - - def versionsArtifact(module: Module): Option[Artifact] = - if (ivyLike) None - else { - val path = ( - module.organization.split('.').toSeq ++ Seq( - module.name, - "maven-metadata.xml" - ) - ) .map(encodeURIComponent) - - val artifact = - Artifact( - path.mkString("/"), - Map.empty, - Map.empty, - Attributes("pom", "") - ) - .withDefaultChecksums - - Some(artifact) - } - - def snapshotVersioningArtifact( - module: Module, - version: String - ): Option[Artifact] = - if (ivyLike) None - else { - val path = ( - module.organization.split('.').toSeq ++ Seq( - module.name, - version, - "maven-metadata.xml" - ) - ) .map(encodeURIComponent) - - val artifact = - Artifact( - path.mkString("/"), - Map.empty, - Map.empty, - Attributes("pom", "") - ) - .withDefaultChecksums - - Some(artifact) - } - - def versions( - module: Module, - cachePolicy: CachePolicy = CachePolicy.Default - ): EitherT[Task, String, Versions] = { - - EitherT( - versionsArtifact(module) match { - case None => Task.now(-\/("Not supported")) - case Some(artifact) => - fetch(artifact, cachePolicy) - .run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - versions <- Xml.versions(xml) - } yield versions - ) - } - ) - } - - def snapshotVersioning( - module: Module, - version: String, - cachePolicy: CachePolicy = CachePolicy.Default - ): EitherT[Task, String, SnapshotVersioning] = { - - EitherT( - snapshotVersioningArtifact(module, version) match { - case None => Task.now(-\/("Not supported")) - case Some(artifact) => - fetch(artifact, cachePolicy) - .run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") - snapshotVersioning <- Xml.snapshotVersioning(xml) - } yield snapshotVersioning - ) - } - ) - } - - def findNoInterval( - module: Module, - version: String, - cachePolicy: CachePolicy - ): EitherT[Task, String, Project] = - EitherT{ - def withSnapshotVersioning = - snapshotVersioning(module, version, cachePolicy) - .flatMap { snapshotVersioning => - val versioningOption = - mavenVersioning(snapshotVersioning, "", "jar") - .orElse(mavenVersioning(snapshotVersioning, "", "")) - - versioningOption match { - case None => - EitherT[Task, String, Project]( - Task.now(-\/("No snapshot versioning value found")) - ) - case versioning @ Some(_) => - findVersioning(module, version, versioning, cachePolicy) - .map(_.copy(snapshotVersioning = Some(snapshotVersioning))) - } - } - - findVersioning(module, version, None, cachePolicy) - .run - .flatMap{ eitherProj => - if (eitherProj.isLeft) - withSnapshotVersioning - .run - .map(eitherProj0 => - if (eitherProj0.isLeft) - eitherProj - else - eitherProj0 - ) - else - Task.now(eitherProj) - } - } - - def findVersioning( - module: Module, - version: String, - versioningValue: Option[String], - cachePolicy: CachePolicy - ): EitherT[Task, String, Project] = { - - EitherT { - fetch(projectArtifact(module, version, versioningValue), cachePolicy) - .run - .map(eitherStr => - for { - str <- eitherStr - xml <- \/.fromEither(compatibility.xmlParse(str)) - _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") - proj <- Xml.project(xml) - } yield proj - ) - } - } - - def find( - module: Module, - version: String - )(implicit - cachePolicy: CachePolicy - ): EitherT[Task, String, (Artifact.Source, Project)] = { - - Parse.versionInterval(version) - .filter(_.isValid) match { - case None => - findNoInterval(module, version, cachePolicy).map((source, _)) - case Some(itv) => - versions(module, cachePolicy) - .flatMap { versions0 => - val eitherVersion = { - val release = Version(versions0.release) - - if (itv.contains(release)) \/-(versions0.release) - else { - val inInterval = versions0.available - .map(Version(_)) - .filter(itv.contains) - - if (inInterval.isEmpty) -\/(s"No version found for $version") - else \/-(inInterval.max.repr) - } - } - - eitherVersion match { - case -\/(reason) => EitherT[Task, String, (Artifact.Source, Project)](Task.now(-\/(reason))) - case \/-(version0) => - findNoInterval(module, version0, cachePolicy) - .map(_.copy(versions = Some(versions0))) - .map((source, _)) - } - } - } - } - -} - -sealed trait CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] -} - -object CachePolicy { - def saving[E,T]( - remote: => Task[E \/ T] )( - save: T => Task[Unit] - ): Task[E \/ T] = { - for { - res <- remote - _ <- res.fold(_ => Task.now(()), t => save(t)) - } yield res - } - - case object Default extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local - .flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res)) - } - case object LocalOnly extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local - } - case object ForceDownload extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - remote(None) - } -} diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala new file mode 100644 index 000000000..3ea96fc81 --- /dev/null +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -0,0 +1,279 @@ +package coursier.maven + +import coursier.core._ +import coursier.core.compatibility.encodeURIComponent + +import scala.language.higherKinds +import scalaz._ + +object MavenRepository { + + def ivyLikePath( + org: String, + name: String, + version: String, + subDir: String, + baseSuffix: String, + ext: String + ) = + Seq( + org, + name, + version, + subDir, + s"$name$baseSuffix.$ext" + ) + + def mavenVersioning( + snapshotVersioning: SnapshotVersioning, + classifier: String, + extension: String + ): Option[String] = + snapshotVersioning + .snapshotVersions + .find(v => v.classifier == classifier && v.extension == extension) + .map(_.value) + .filter(_.nonEmpty) + +} + +case class MavenRepository( + root: String, + ivyLike: Boolean = false +) extends Repository { + + import Repository._ + import MavenRepository._ + + val root0 = if (root.endsWith("/")) root else root + "/" + val source = MavenSource(root0, ivyLike) + + def projectArtifact( + module: Module, + version: String, + versioningValue: Option[String] + ): Artifact = { + + val path = ( + if (ivyLike) + ivyLikePath( + module.organization, + module.name, + versioningValue getOrElse version, + "poms", + "", + "pom" + ) + else + module.organization.split('.').toSeq ++ Seq( + module.name, + version, + s"${module.name}-${versioningValue getOrElse version}.pom" + ) + ) .map(encodeURIComponent) + + Artifact( + root0 + path.mkString("/"), + Map.empty, + Map.empty, + Attributes("pom", "") + ) + .withDefaultChecksums + .withDefaultSignature + } + + def versionsArtifact(module: Module): Option[Artifact] = + if (ivyLike) None + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + "maven-metadata.xml" + ) + ) .map(encodeURIComponent) + + val artifact = + Artifact( + root0 + path.mkString("/"), + Map.empty, + Map.empty, + Attributes("pom", "") + ) + .withDefaultChecksums + .withDefaultChecksums + + Some(artifact) + } + + def snapshotVersioningArtifact( + module: Module, + version: String + ): Option[Artifact] = + if (ivyLike) None + else { + val path = ( + module.organization.split('.').toSeq ++ Seq( + module.name, + version, + "maven-metadata.xml" + ) + ) .map(encodeURIComponent) + + val artifact = + Artifact( + root0 + path.mkString("/"), + Map.empty, + Map.empty, + Attributes("pom", "") + ) + .withDefaultChecksums + .withDefaultSignature + + Some(artifact) + } + + def versions[F[_]]( + module: Module, + fetch: Repository.Fetch[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, Versions] = + EitherT( + versionsArtifact(module) match { + case None => F.point(-\/("Not supported")) + case Some(artifact) => + F.map(fetch(artifact).run)(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") + versions <- Pom.versions(xml) + } yield versions + ) + } + ) + + def snapshotVersioning[F[_]]( + module: Module, + version: String, + fetch: Repository.Fetch[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, SnapshotVersioning] = { + + EitherT( + snapshotVersioningArtifact(module, version) match { + case None => F.point(-\/("Not supported")) + case Some(artifact) => + F.map(fetch(artifact).run)(eitherStr => + for { + str <- eitherStr + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found") + snapshotVersioning <- Pom.snapshotVersioning(xml) + } yield snapshotVersioning + ) + } + ) + } + + def findNoInterval[F[_]]( + module: Module, + version: String, + fetch: Repository.Fetch[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, Project] = + EitherT { + def withSnapshotVersioning = + snapshotVersioning(module, version, fetch).flatMap { snapshotVersioning => + val versioningOption = + mavenVersioning(snapshotVersioning, "", "jar") + .orElse(mavenVersioning(snapshotVersioning, "", "")) + + versioningOption match { + case None => + EitherT[F, String, Project]( + F.point(-\/("No snapshot versioning value found")) + ) + case versioning @ Some(_) => + findVersioning(module, version, versioning, fetch) + .map(_.copy(snapshotVersioning = Some(snapshotVersioning))) + } + } + + F.bind(findVersioning(module, version, None, fetch).run) { eitherProj => + if (eitherProj.isLeft) + F.map(withSnapshotVersioning.run)(eitherProj0 => + if (eitherProj0.isLeft) + eitherProj + else + eitherProj0 + ) + else + F.point(eitherProj) + } + } + + def findVersioning[F[_]]( + module: Module, + version: String, + versioningValue: Option[String], + fetch: Repository.Fetch[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, Project] = { + + fetch(projectArtifact(module, version, versioningValue)).flatMap { str => + EitherT { + F.point { + (for { + xml <- \/.fromEither(compatibility.xmlParse(str)) + _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found") + proj <- Pom.project(xml) + } yield proj): (String \/ Project) + } + } + } + } + + def find[F[_]]( + module: Module, + version: String, + fetch: Repository.Fetch[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, (Artifact.Source, Project)] = { + + Parse.versionInterval(version) + .filter(_.isValid) match { + case None => + findNoInterval(module, version, fetch).map((source, _)) + case Some(itv) => + versions(module, fetch).flatMap { versions0 => + val eitherVersion = { + val release = Version(versions0.release) + + if (itv.contains(release)) \/-(versions0.release) + else { + val inInterval = versions0.available + .map(Version(_)) + .filter(itv.contains) + + if (inInterval.isEmpty) -\/(s"No version found for $version") + else \/-(inInterval.max.repr) + } + } + + eitherVersion match { + case -\/(reason) => EitherT[F, String, (Artifact.Source, Project)](F.point(-\/(reason))) + case \/-(version0) => + findNoInterval(module, version0, fetch) + .map(_.copy(versions = Some(versions0))) + .map((source, _)) + } + } + } + } + +} diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala new file mode 100644 index 000000000..02c780d3f --- /dev/null +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -0,0 +1,79 @@ +package coursier.maven + +import coursier.core._ + +case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { + import Repository._ + import MavenRepository._ + + def artifacts( + dependency: Dependency, + project: Project + ): Seq[Artifact] = { + + def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) = + ivyLikePath( + dependency.module.organization, + dependency.module.name, + project.version, + subDir, + baseSuffix, + ext + ) + + val path = + if (ivyLike) + ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`) + else { + val versioning = + project + .snapshotVersioning + .flatMap(versioning => + mavenVersioning(versioning, dependency.attributes.classifier, dependency.attributes.`type`) + ) + + dependency.module.organization.split('.').toSeq ++ Seq( + dependency.module.name, + project.version, + s"${dependency.module.name}-${versioning getOrElse project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}" + ) + } + + var artifact = + Artifact( + root + path.mkString("/"), + Map.empty, + Map.empty, + dependency.attributes + ) + .withDefaultChecksums + + if (dependency.attributes.`type` == "jar") { + artifact = artifact.withDefaultSignature + + // FIXME Snapshot versioning of sources and javadoc is not taken into account here. + // Will be ok if it's the same as the main JAR though. + + artifact = + if (ivyLike) { + val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/") + val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/") + + artifact + .copy( + extra = artifact.extra ++ Map( + "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? + .withDefaultChecksums + .withDefaultSignature, + "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above + .withDefaultChecksums + .withDefaultSignature + )) + } else + artifact + .withJavadocSources + } + + Seq(artifact) + } +} diff --git a/core/shared/src/main/scala/coursier/core/Xml.scala b/core/shared/src/main/scala/coursier/maven/Pom.scala similarity index 95% rename from core/shared/src/main/scala/coursier/core/Xml.scala rename to core/shared/src/main/scala/coursier/maven/Pom.scala index 9bd94700e..8033498fa 100644 --- a/core/shared/src/main/scala/coursier/core/Xml.scala +++ b/core/shared/src/main/scala/coursier/maven/Pom.scala @@ -1,28 +1,11 @@ -package coursier.core +package coursier.maven + +import coursier.core._ import scalaz._ -object Xml { - - /** A representation of an XML node/document, with different implementations on the JVM and JS */ - trait Node { - def label: String - def child: Seq[Node] - def isText: Boolean - def textContent: String - def isElement: Boolean - } - - object Node { - val empty: Node = - new Node { - val isText = false - val isElement = false - val child = Nil - val label = "" - val textContent = "" - } - } +object Pom { + import coursier.util.Xml._ object Text { def unapply(n: Node): Option[String] = @@ -340,7 +323,7 @@ object Xml { .traverseU(snapshotVersion) } yield SnapshotVersioning( Module(organization, name), - version, + version, latest, release, timestamp, @@ -350,5 +333,4 @@ object Xml { snapshotVersions ) } - } diff --git a/core/shared/src/main/scala/coursier/package.scala b/core/shared/src/main/scala/coursier/package.scala index e9f30981f..d060d0b4b 100644 --- a/core/shared/src/main/scala/coursier/package.scala +++ b/core/shared/src/main/scala/coursier/package.scala @@ -1,8 +1,6 @@ -import scalaz.{ -\/, \/- } -import scalaz.concurrent.Task /** - * Pulls definitions from coursier.core, sometimes with default arguments. + * Mainly pulls definitions from coursier.core, sometimes with default arguments. */ package object coursier { @@ -37,10 +35,10 @@ package object coursier { } type Project = core.Project - val Project: core.Project.type = core.Project + val Project = core.Project type Profile = core.Profile - val Profile: core.Profile.type = core.Profile + val Profile = core.Profile type Module = core.Module object Module { @@ -51,13 +49,13 @@ package object coursier { type ModuleVersion = (core.Module, String) type Scope = core.Scope - val Scope: core.Scope.type = core.Scope - - type CachePolicy = core.CachePolicy - val CachePolicy: core.CachePolicy.type = core.CachePolicy + val Scope = core.Scope type Repository = core.Repository - val Repository: core.Repository.type = core.Repository + val Repository = core.Repository + + type MavenRepository = maven.MavenRepository + val MavenRepository = maven.MavenRepository type Resolution = core.Resolution object Resolution { @@ -83,54 +81,14 @@ package object coursier { } type Artifact = core.Artifact - val Artifact: core.Artifact.type = core.Artifact + val Artifact = core.Artifact type ResolutionProcess = core.ResolutionProcess - val ResolutionProcess: core.ResolutionProcess.type = core.ResolutionProcess + val ResolutionProcess = core.ResolutionProcess implicit class ResolutionExtensions(val underlying: Resolution) extends AnyVal { def process: ResolutionProcess = ResolutionProcess(underlying) } - def fetch( - repositories: Seq[core.Repository] - )(implicit - cachePolicy: CachePolicy - ): ResolutionProcess.Fetch[Task] = { - - modVers => - Task.gatherUnordered( - modVers - .map {case (module, version) => - Repository.find(repositories, module, version) - .run - .map((module, version) -> _) - } - ) - } - - implicit def fetchLocalFirst( - repositories: Seq[core.Repository] - )(implicit - cachePolicy: CachePolicy - ): ResolutionProcess.Fetch[Task] = { - - modVers => - Task.gatherUnordered( - modVers - .map {case (module, version) => - def attempt(cachePolicy: CachePolicy) = - Repository.find(repositories, module, version)(cachePolicy) - .run - .map((module, version) -> _) - - attempt(CachePolicy.LocalOnly).flatMap { - case v @ (_, \/-(_)) => Task.now(v) - case (_, -\/(_)) => attempt(cachePolicy) - } - } - ) - } - } diff --git a/core/shared/src/main/scala/coursier/util/Xml.scala b/core/shared/src/main/scala/coursier/util/Xml.scala new file mode 100644 index 000000000..5518a58d2 --- /dev/null +++ b/core/shared/src/main/scala/coursier/util/Xml.scala @@ -0,0 +1,25 @@ +package coursier.util + +object Xml { + + /** A representation of an XML node/document, with different implementations on JVM and JS */ + trait Node { + def label: String + def child: Seq[Node] + def isText: Boolean + def textContent: String + def isElement: Boolean + } + + object Node { + val empty: Node = + new Node { + val isText = false + val isElement = false + val child = Nil + val label = "" + val textContent = "" + } + } + +} diff --git a/fetch-js/src/main/scala/coursier/Fetch.scala b/fetch-js/src/main/scala/coursier/Fetch.scala new file mode 100644 index 000000000..3e7d220d5 --- /dev/null +++ b/fetch-js/src/main/scala/coursier/Fetch.scala @@ -0,0 +1,26 @@ +package coursier + +import scalaz.concurrent.Task + +object Fetch { + + implicit def default( + repositories: Seq[core.Repository] + ): ResolutionProcess.Fetch[Task] = + apply(repositories, Platform.artifact) + + def apply( + repositories: Seq[core.Repository], + fetch: Repository.Fetch[Task] + ): ResolutionProcess.Fetch[Task] = { + + modVers => Task.gatherUnordered( + modVers.map { case (module, version) => + Repository.find(repositories, module, version, fetch) + .run + .map((module, version) -> _) + } + ) + } + +} diff --git a/core/js/src/main/scala/coursier/core/MavenRepository.scala b/fetch-js/src/main/scala/coursier/Platform.scala similarity index 78% rename from core/js/src/main/scala/coursier/core/MavenRepository.scala rename to fetch-js/src/main/scala/coursier/Platform.scala index 67cf122a7..799c20fb5 100644 --- a/core/js/src/main/scala/coursier/core/MavenRepository.scala +++ b/fetch-js/src/main/scala/coursier/Platform.scala @@ -1,18 +1,17 @@ package coursier -package core import org.scalajs.dom.raw.{ Event, XMLHttpRequest } import scala.concurrent.{ ExecutionContext, Promise, Future } -import scalaz.{ -\/, \/-, EitherT } -import scalaz.concurrent.Task import scala.scalajs.js import js.Dynamic.{ global => g } import scala.scalajs.js.timers._ +import scalaz.concurrent.Task +import scalaz.{ -\/, \/-, EitherT } -object MavenRepository { +object Platform { def encodeURIComponent(s: String): String = g.encodeURIComponent(s).asInstanceOf[String] @@ -76,37 +75,34 @@ object MavenRepository { p.future } + val artifact: Repository.Fetch[Task] = { artifact => + EitherT( + Task { implicit ec => + get(artifact.url) + .map(\/-(_)) + .recover { case e: Exception => + -\/(e.getMessage) + } + } + ) + } + trait Logger { def fetching(url: String): Unit def fetched(url: String): Unit def other(url: String, msg: String): Unit } -} - -case class MavenRepository( - root: String, - ivyLike: Boolean = false, - logger: Option[MavenRepository.Logger] = None -) extends BaseMavenRepository(root, ivyLike) { - - - def fetch( - artifact: Artifact, - cachePolicy: CachePolicy - ): EitherT[Task, String, String] = { - - val url0 = root + artifact.url - + def artifactWithLogger(logger: Logger): Repository.Fetch[Task] = { artifact => EitherT( Task { implicit ec => - Future(logger.foreach(_.fetching(url0))) - .flatMap(_ => MavenRepository.get(url0)) - .map{ s => logger.foreach(_.fetched(url0)); \/-(s) } - .recover{case e: Exception => - logger.foreach(_.other(url0, e.getMessage)) - -\/(e.getMessage) - } + Future(logger.fetching(artifact.url)) + .flatMap(_ => get(artifact.url)) + .map { s => logger.fetched(artifact.url); \/-(s) } + .recover { case e: Exception => + logger.other(artifact.url, e.getMessage) + -\/(e.getMessage) + } } ) } diff --git a/core/js/src/main/scala/scalaz/concurrent/package.scala b/fetch-js/src/main/scala/scalaz/concurrent/package.scala similarity index 100% rename from core/js/src/main/scala/scalaz/concurrent/package.scala rename to fetch-js/src/main/scala/scalaz/concurrent/package.scala diff --git a/files/src/main/scala/coursier/Cache.scala b/files/src/main/scala/coursier/Cache.scala index b33f9151e..b78e578b4 100644 --- a/files/src/main/scala/coursier/Cache.scala +++ b/files/src/main/scala/coursier/Cache.scala @@ -1,9 +1,6 @@ package coursier -import java.io.{PrintWriter, File} - -import coursier.core.MavenRepository - +import java.io.{ File, PrintWriter } import scala.io.Source object Cache { @@ -93,10 +90,9 @@ case class Cache(cache: File) { .flatMap { f => val name = f.getName val lines = Source.fromFile(f).getLines().toList - mavenRepository(lines) - .map(repo => - (name, repo.copy(cache = Some(new File(metadataBase, name))), (repo.root, new File(fileBase, name))) - ) + mavenRepository(lines).map(repo => + (name, repo, (repo.root, new File(fileBase, name))) + ) } def map(): Map[String, (MavenRepository, (String, File))] = diff --git a/files/src/main/scala/coursier/CachePolicy.scala b/files/src/main/scala/coursier/CachePolicy.scala new file mode 100644 index 000000000..10c76392f --- /dev/null +++ b/files/src/main/scala/coursier/CachePolicy.scala @@ -0,0 +1,49 @@ +package coursier + +import scalaz.\/ +import scalaz.concurrent.Task + +sealed trait CachePolicy { + def apply[T]( + tryRemote: T => Boolean )( + local: => Task[T] )( + remote: Option[T] => Task[T] + ): Task[T] +} + +object CachePolicy { + def saving[E,T]( + remote: => Task[E \/ T] )( + save: T => Task[Unit] + ): Task[E \/ T] = { + for { + res <- remote + _ <- res.fold(_ => Task.now(()), t => save(t)) + } yield res + } + + case object Default extends CachePolicy { + def apply[T]( + tryRemote: T => Boolean )( + local: => Task[T] )( + remote: Option[T] => Task[T] + ): Task[T] = + local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res)) + } + case object LocalOnly extends CachePolicy { + def apply[T]( + tryRemote: T => Boolean )( + local: => Task[T] )( + remote: Option[T] => Task[T] + ): Task[T] = + local + } + case object ForceDownload extends CachePolicy { + def apply[T]( + tryRemote: T => Boolean )( + local: => Task[T] )( + remote: Option[T] => Task[T] + ): Task[T] = + remote(None) + } +} diff --git a/files/src/main/scala/coursier/Fetch.scala b/files/src/main/scala/coursier/Fetch.scala new file mode 100644 index 000000000..3e7d220d5 --- /dev/null +++ b/files/src/main/scala/coursier/Fetch.scala @@ -0,0 +1,26 @@ +package coursier + +import scalaz.concurrent.Task + +object Fetch { + + implicit def default( + repositories: Seq[core.Repository] + ): ResolutionProcess.Fetch[Task] = + apply(repositories, Platform.artifact) + + def apply( + repositories: Seq[core.Repository], + fetch: Repository.Fetch[Task] + ): ResolutionProcess.Fetch[Task] = { + + modVers => Task.gatherUnordered( + modVers.map { case (module, version) => + Repository.find(repositories, module, version, fetch) + .run + .map((module, version) -> _) + } + ) + } + +} diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index 23c74fd85..e0d074035 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -14,7 +14,6 @@ import java.io._ case class Files( cache: Seq[(String, File)], tmp: () => File, - logger: Option[Files.Logger] = None, concurrentDownloadCount: Int = Files.defaultConcurrentDownloadCount ) { @@ -22,18 +21,18 @@ case class Files( Executors.newFixedThreadPool(concurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) def withLocal(artifact: Artifact): Artifact = { - val isLocal = - artifact.url.startsWith("file:/") && - artifact.checksumUrls.values.forall(_.startsWith("file:/")) - def local(url: String) = if (url.startsWith("file:///")) url.stripPrefix("file://") else if (url.startsWith("file:/")) url.stripPrefix("file:") else - cache.find{case (base, _) => url.startsWith(base)} match { - case None => ??? + cache.find { case (base, _) => url.startsWith(base) } match { + case None => + // FIXME Means we were handed an artifact from repositories other than the known ones + println(cache.mkString("\n")) + println(url) + ??? case Some((base, cacheDir)) => cacheDir + "/" + url.stripPrefix(base) } @@ -55,7 +54,8 @@ case class Files( def download( artifact: Artifact, - withChecksums: Boolean = true + withChecksums: Boolean = true, + logger: Option[Files.Logger] = None )(implicit cachePolicy: CachePolicy, pool: ExecutorService = defaultPool @@ -75,10 +75,10 @@ case class Files( } - def locally(file: File) = + def locally(file: File, url: String) = Task { if (file.exists()) { - logger.foreach(_.foundLocally(file)) + logger.foreach(_.foundLocally(url, file)) \/-(file) } else -\/(FileError.NotFound(file.toString): FileError) @@ -94,8 +94,17 @@ case class Files( logger.foreach(_.downloadingArtifact(url)) - val url0 = new URL(url) - val in = new BufferedInputStream(url0.openStream(), Files.bufferSize) + val conn = new URL(url).openConnection() // FIXME Should this be closed? + // Dummy user-agent instead of the default "Java/...", + // so that we are not returned incomplete/erroneous metadata + // (Maven 2 compatibility? - happens for snapshot versioning metadata, + // this is SO FUCKING CRAZY) + conn.setRequestProperty("User-Agent", "") + + for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) + logger.foreach(_.downloadLength(url, len)) + + val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize) val result = try { @@ -110,15 +119,16 @@ case class Files( val b = Array.fill[Byte](Files.bufferSize)(0) @tailrec - def helper(): Unit = { + def helper(count: Long): Unit = { val read = in.read(b) if (read >= 0) { out.write(b, 0, read) - helper() + logger.foreach(_.downloadProgress(url, count + read)) + helper(count + read) } } - helper() + helper(0L) \/-(file) } } @@ -130,6 +140,9 @@ case class Files( } finally out.close() } finally in.close() + for (lastModified <- Option(conn.getLastModified).filter(_ > 0L)) + file.setLastModified(lastModified) + logger.foreach(_.downloadedArtifact(url, success = true)) result } @@ -141,19 +154,26 @@ case class Files( val tasks = - for ((f, url) <- pairs) yield + for ((f, url) <- pairs) yield { + val file = new File(f) + if (url != ("file:" + f) && url != ("file://" + f)) { assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url") - val file = new File(f) cachePolicy[FileError \/ File]( - _.isLeft)( - locally(file))( + _.isLeft )( + locally(file, url) )( _ => remote(file, url) ).map(e => (file, url) -> e.map(_ => ())) - } else { - val file = new File(f) - Task.now(((file, url), \/-(()))) - } + } else + Task { + (file, url) -> { + if (file.exists()) + \/-(()) + else + -\/(FileError.NotFound(file.toString)) + } + } + } Nondeterminism[Task].gather(tasks) } @@ -200,40 +220,60 @@ case class Files( def file( artifact: Artifact, - checksum: Option[String] = Some("SHA-1") + checksum: Option[String] = Some("SHA-1"), + logger: Option[Files.Logger] = None )(implicit cachePolicy: CachePolicy, pool: ExecutorService = defaultPool ): EitherT[Task, FileError, File] = - EitherT{ - val res = - download(artifact) - .map(results => - results.head._2.map(_ => results.head._1._1) - ) + EitherT { + val res = download(artifact, withChecksums = checksum.nonEmpty, logger = logger).map { + results => + val ((f, _), res) = results.head + res.map(_ => f) + } checksum.fold(res) { sumType => - res - .flatMap{ - case err @ -\/(_) => Task.now(err) - case \/-(f) => - validateChecksum(artifact, sumType) - .map(_.map(_ => f)) - } + res.flatMap { + case err @ -\/(_) => Task.now(err) + case \/-(f) => + validateChecksum(artifact, sumType) + .map(_.map(_ => f)) + } } } + def fetch( + checksum: Option[String] = Some("SHA-1"), + logger: Option[Files.Logger] = None + )(implicit + cachePolicy: CachePolicy, + pool: ExecutorService = defaultPool + ): Repository.Fetch[Task] = { + artifact => + file(artifact, checksum = checksum, logger = logger)(cachePolicy).leftMap(_.message).map { f => + // FIXME Catch error here? + scala.io.Source.fromFile(f)("UTF-8").mkString + } + } + } object Files { - + + lazy val ivy2Local = MavenRepository( + new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, + ivyLike = true + ) + val defaultConcurrentDownloadCount = 6 - // FIXME This kind of side-effecting API is lame, we should aim at a more functional one. trait Logger { - def foundLocally(f: File): Unit - def downloadingArtifact(url: String): Unit - def downloadedArtifact(url: String, success: Boolean): Unit + def foundLocally(url: String, f: File): Unit = {} + def downloadingArtifact(url: String): Unit = {} + def downloadLength(url: String, length: Long): Unit = {} + def downloadProgress(url: String, downloaded: Long): Unit = {} + def downloadedArtifact(url: String, success: Boolean): Unit = {} } var bufferSize = 1024*1024 @@ -276,14 +316,26 @@ object Files { } -sealed trait FileError +sealed trait FileError { + def message: String +} object FileError { - case class DownloadError(message: String) extends FileError - case class NotFound(file: String) extends FileError - case class Locked(file: String) extends FileError - case class ChecksumNotFound(sumType: String, file: String) extends FileError - case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError + case class DownloadError(message0: String) extends FileError { + def message = s"Download error: $message0" + } + case class NotFound(file: String) extends FileError { + def message = s"$file: not found" + } + case class Locked(file: String) extends FileError { + def message = s"$file: locked" + } + case class ChecksumNotFound(sumType: String, file: String) extends FileError { + def message = s"$file: $sumType checksum not found" + } + case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError { + def message = s"$file: $sumType checksum validation failed" + } } diff --git a/files/src/main/scala/coursier/Platform.scala b/files/src/main/scala/coursier/Platform.scala new file mode 100644 index 000000000..c376fefe3 --- /dev/null +++ b/files/src/main/scala/coursier/Platform.scala @@ -0,0 +1,56 @@ +package coursier + +import java.io._ +import java.net.URL + +import scalaz._ +import scalaz.concurrent.Task + +object Platform { + + def readFullySync(is: InputStream) = { + val buffer = new ByteArrayOutputStream() + val data = Array.ofDim[Byte](16384) + + var nRead = is.read(data, 0, data.length) + while (nRead != -1) { + buffer.write(data, 0, nRead) + nRead = is.read(data, 0, data.length) + } + + buffer.flush() + buffer.toByteArray + } + + def readFully(is: => InputStream) = + Task { + \/.fromTryCatchNonFatal { + val is0 = is + val b = + try readFullySync(is0) + finally is0.close() + + new String(b, "UTF-8") + } .leftMap{ + case e: java.io.FileNotFoundException => + s"Not found: ${e.getMessage}" + case e => + s"$e: ${e.getMessage}" + } + } + + val artifact: Repository.Fetch[Task] = { artifact => + EitherT { + val url = new URL(artifact.url) + + val conn = url.openConnection() + // Dummy user-agent instead of the default "Java/...", + // so that we are not returned incomplete/erroneous metadata + // (Maven 2 compatibility? - happens for snapshot versioning metadata, + // this is SO FUCKING CRAZY) + conn.setRequestProperty("User-Agent", "") + readFully(conn.getInputStream()) + } + } + +} diff --git a/core/js/src/test/scala/coursier/test/JsTests.scala b/tests/js/src/test/scala/coursier/test/JsTests.scala similarity index 70% rename from core/js/src/test/scala/coursier/test/JsTests.scala rename to tests/js/src/test/scala/coursier/test/JsTests.scala index c59f1dc80..55bc44213 100644 --- a/core/js/src/test/scala/coursier/test/JsTests.scala +++ b/tests/js/src/test/scala/coursier/test/JsTests.scala @@ -1,12 +1,11 @@ package coursier package test -import coursier.core.{Repository, MavenRepository} import coursier.test.compatibility._ import utest._ -import scala.concurrent.{Future, Promise} +import scala.concurrent.{ Future, Promise } object JsTests extends TestSuite { @@ -18,7 +17,7 @@ object JsTests extends TestSuite { } 'get{ - MavenRepository.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") + Platform.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom") .map(core.compatibility.xmlParse) .map{ xml => assert(xml.right.toOption.exists(_.label == "project")) @@ -26,10 +25,8 @@ object JsTests extends TestSuite { } 'getProj{ - implicit val cachePolicy = CachePolicy.Default - - Repository.mavenCentral - .find(Module("ch.qos.logback", "logback-classic"), "1.1.3") + MavenRepository("https://repo1.maven.org/maven2/") + .find(Module("ch.qos.logback", "logback-classic"), "1.1.3", Platform.artifact) .map{case (_, proj) => assert(proj.parent == Some(Module("ch.qos.logback", "logback-parent"), "1.1.3")) } diff --git a/core/js/src/test/scala/coursier/test/compatibility/package.scala b/tests/js/src/test/scala/coursier/test/compatibility/package.scala similarity index 90% rename from core/js/src/test/scala/coursier/test/compatibility/package.scala rename to tests/js/src/test/scala/coursier/test/compatibility/package.scala index 38099453c..fb2d0da9c 100644 --- a/core/js/src/test/scala/coursier/test/compatibility/package.scala +++ b/tests/js/src/test/scala/coursier/test/compatibility/package.scala @@ -13,7 +13,7 @@ package object compatibility { def textResource(path: String)(implicit ec: ExecutionContext): Future[String] = { val p = Promise[String]() - fs.readFile("core/shared/src/test/resources/" + path, "utf-8", { + fs.readFile("tests/shared/src/test/resources/" + path, "utf-8", { (err: js.Dynamic, data: js.Dynamic) => if (err == null) p.success(data.asInstanceOf[String]) else p.failure(new Exception(err.toString)) diff --git a/core/jvm/src/test/scala/coursier/test/IvyLocalTests.scala b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala similarity index 78% rename from core/jvm/src/test/scala/coursier/test/IvyLocalTests.scala rename to tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala index f9b820848..ac0c31865 100644 --- a/core/jvm/src/test/scala/coursier/test/IvyLocalTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala @@ -1,7 +1,6 @@ package coursier.test -import coursier.Module -import coursier.core.Repository +import coursier.{ Module, Files } import utest._ object IvyLocalTests extends TestSuite { @@ -11,7 +10,7 @@ object IvyLocalTests extends TestSuite { // Assume this module (and the sub-projects it depends on) is published locally CentralTests.resolutionCheck( Module("com.github.alexarchambault", "coursier_2.11"), "0.1.0-SNAPSHOT", - Some(Repository.ivy2Local)) + Some(Files.ivy2Local)) } } diff --git a/core/jvm/src/test/scala/coursier/test/compatibility/package.scala b/tests/jvm/src/test/scala/coursier/test/compatibility/package.scala similarity index 85% rename from core/jvm/src/test/scala/coursier/test/compatibility/package.scala rename to tests/jvm/src/test/scala/coursier/test/compatibility/package.scala index a08aa54e4..b257db280 100644 --- a/core/jvm/src/test/scala/coursier/test/compatibility/package.scala +++ b/tests/jvm/src/test/scala/coursier/test/compatibility/package.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.core.MavenRepository +import coursier.Platform import scala.concurrent.{ExecutionContext, Future} import scalaz.concurrent.Task @@ -19,7 +19,7 @@ package object compatibility { .getResource(path) .openStream() - new String(MavenRepository.readFullySync(is), "UTF-8") + new String(Platform.readFullySync(is), "UTF-8") } } diff --git a/core/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0 b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0 similarity index 100% rename from core/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0 rename to tests/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0 diff --git a/core/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0.jcabi b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0.jcabi similarity index 100% rename from core/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0.jcabi rename to tests/shared/src/test/resources/resolutions/com.github.alexarchambault/argonaut-shapeless_6.1_2.11/0.2.0.jcabi diff --git a/core/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT similarity index 74% rename from core/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT rename to tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT index ed11fd041..cc4c8e9b1 100644 --- a/core/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT +++ b/tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT @@ -2,6 +2,4 @@ com.github.alexarchambault:coursier_2.11:jar:0.1.0-SNAPSHOT org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4 org.scala-lang.modules:scala-xml_2.11:jar:1.0.4 org.scala-lang:scala-library:jar:2.11.7 -org.scalaz:scalaz-concurrent_2.11:jar:7.1.2 org.scalaz:scalaz-core_2.11:jar:7.1.2 -org.scalaz:scalaz-effect_2.11:jar:7.1.2 diff --git a/core/shared/src/test/resources/resolutions/com.github.fommil/java-logging/1.2-SNAPSHOT b/tests/shared/src/test/resources/resolutions/com.github.fommil/java-logging/1.2-SNAPSHOT similarity index 100% rename from core/shared/src/test/resources/resolutions/com.github.fommil/java-logging/1.2-SNAPSHOT rename to tests/shared/src/test/resources/resolutions/com.github.fommil/java-logging/1.2-SNAPSHOT diff --git a/core/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 b/tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 similarity index 100% rename from core/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 rename to tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1 diff --git a/core/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1.jcabi b/tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1.jcabi similarity index 100% rename from core/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1.jcabi rename to tests/shared/src/test/resources/resolutions/org.apache.spark/spark-core_2.11/1.3.1.jcabi diff --git a/core/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala similarity index 96% rename from core/shared/src/test/scala/coursier/test/CentralTests.scala rename to tests/shared/src/test/scala/coursier/test/CentralTests.scala index 6747537e0..751397d13 100644 --- a/core/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -1,20 +1,18 @@ package coursier package test -import coursier.core.{ Repository, MavenRepository } import utest._ import scala.async.Async.{ async, await } +import coursier.Fetch.default import coursier.test.compatibility._ object CentralTests extends TestSuite { val repositories = Seq[Repository]( - Repository.mavenCentral + MavenRepository("https://repo1.maven.org/maven2/") ) - implicit val cachePolicy = CachePolicy.Default - def resolve( deps: Set[Dependency], filter: Option[Dependency => Boolean] = None, diff --git a/core/shared/src/test/scala/coursier/test/ExclusionsTests.scala b/tests/shared/src/test/scala/coursier/test/ExclusionsTests.scala similarity index 100% rename from core/shared/src/test/scala/coursier/test/ExclusionsTests.scala rename to tests/shared/src/test/scala/coursier/test/ExclusionsTests.scala diff --git a/core/shared/src/test/scala/coursier/test/PomParsingTests.scala b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala similarity index 91% rename from core/shared/src/test/scala/coursier/test/PomParsingTests.scala rename to tests/shared/src/test/scala/coursier/test/PomParsingTests.scala index ae56eab8a..6bea7b696 100644 --- a/core/shared/src/test/scala/coursier/test/PomParsingTests.scala +++ b/tests/shared/src/test/scala/coursier/test/PomParsingTests.scala @@ -4,7 +4,8 @@ package test import utest._ import scalaz._ -import coursier.core.Xml +import coursier.maven.Pom + import coursier.core.compatibility._ object PomParsingTests extends TestSuite { @@ -22,7 +23,7 @@ object PomParsingTests extends TestSuite { val expected = \/-(Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra"))) - val result = Xml.dependency(xmlParse(depNode).right.get) + val result = Pom.dependency(xmlParse(depNode).right.get) assert(result == expected) } @@ -35,7 +36,7 @@ object PomParsingTests extends TestSuite { val expected = \/-(Profile("profile1", None, Profile.Activation(Nil), Nil, Nil, Map.empty)) - val result = Xml.profile(xmlParse(profileNode).right.get) + val result = Pom.profile(xmlParse(profileNode).right.get) assert(result == expected) } @@ -50,7 +51,7 @@ object PomParsingTests extends TestSuite { val expected = \/-(Profile("", Some(true), Profile.Activation(Nil), Nil, Nil, Map.empty)) - val result = Xml.profile(xmlParse(profileNode).right.get) + val result = Pom.profile(xmlParse(profileNode).right.get) assert(result == expected) } @@ -66,7 +67,7 @@ object PomParsingTests extends TestSuite { val expected = \/-(Profile("profile1", Some(true), Profile.Activation(Nil), Nil, Nil, Map.empty)) - val result = Xml.profile(xmlParse(profileNode).right.get) + val result = Pom.profile(xmlParse(profileNode).right.get) assert(result == expected) } @@ -94,7 +95,7 @@ object PomParsingTests extends TestSuite { Map.empty )) - val result = Xml.profile(xmlParse(profileNode).right.get) + val result = Pom.profile(xmlParse(profileNode).right.get) assert(result == expected) } @@ -125,7 +126,7 @@ object PomParsingTests extends TestSuite { Map.empty )) - val result = Xml.profile(xmlParse(profileNode).right.get) + val result = Pom.profile(xmlParse(profileNode).right.get) assert(result == expected) } @@ -148,7 +149,7 @@ object PomParsingTests extends TestSuite { Map("first.prop" -> "value1") )) - val result = Xml.profile(xmlParse(profileNode).right.get) + val result = Pom.profile(xmlParse(profileNode).right.get) assert(result == expected) } @@ -194,7 +195,7 @@ object PomParsingTests extends TestSuite { assert(node.label == "properties") val children = node.child.collect{case elem if elem.isElement => elem} - val props0 = children.toList.traverseU(Xml.property) + val props0 = children.toList.traverseU(Pom.property) assert(props0.isRight) diff --git a/core/shared/src/test/scala/coursier/test/ResolutionTests.scala b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala similarity index 99% rename from core/shared/src/test/scala/coursier/test/ResolutionTests.scala rename to tests/shared/src/test/scala/coursier/test/ResolutionTests.scala index 6313d3fdd..5b17d8e1e 100644 --- a/core/shared/src/test/scala/coursier/test/ResolutionTests.scala +++ b/tests/shared/src/test/scala/coursier/test/ResolutionTests.scala @@ -3,18 +3,16 @@ package test import coursier.core.Repository import utest._ -import scala.async.Async.{async, await} +import scala.async.Async.{ async, await } import coursier.test.compatibility._ object ResolutionTests extends TestSuite { - implicit val cachePolicy = CachePolicy.Default - def resolve0(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = { Resolution(deps, filter = filter) .process - .run(fetch(repositories)) + .run(Fetch.default(repositories)) .runF } diff --git a/core/shared/src/test/scala/coursier/test/TestRepository.scala b/tests/shared/src/test/scala/coursier/test/TestRepository.scala similarity index 67% rename from core/shared/src/test/scala/coursier/test/TestRepository.scala rename to tests/shared/src/test/scala/coursier/test/TestRepository.scala index 18b202e6f..85144d66c 100644 --- a/core/shared/src/test/scala/coursier/test/TestRepository.scala +++ b/tests/shared/src/test/scala/coursier/test/TestRepository.scala @@ -3,16 +3,21 @@ package test import coursier.core._ -import scalaz.EitherT -import scalaz.concurrent.Task +import scalaz.{ Monad, EitherT } import scalaz.Scalaz._ class TestRepository(projects: Map[(Module, String), Project]) extends Repository { val source = new core.Artifact.Source { def artifacts(dependency: Dependency, project: Project) = ??? } - def find(module: Module, version: String)(implicit cachePolicy: CachePolicy) = - EitherT(Task.now( + def find[F[_]]( + module: Module, + version: String, + fetch: Repository.Fetch[F] + )(implicit + F: Monad[F] + ) = + EitherT(F.point( projects.get((module, version)).map((source, _)).toRightDisjunction("Not found") )) } diff --git a/core/shared/src/test/scala/coursier/test/VersionConstraintTests.scala b/tests/shared/src/test/scala/coursier/test/VersionConstraintTests.scala similarity index 100% rename from core/shared/src/test/scala/coursier/test/VersionConstraintTests.scala rename to tests/shared/src/test/scala/coursier/test/VersionConstraintTests.scala diff --git a/core/shared/src/test/scala/coursier/test/VersionIntervalTests.scala b/tests/shared/src/test/scala/coursier/test/VersionIntervalTests.scala similarity index 100% rename from core/shared/src/test/scala/coursier/test/VersionIntervalTests.scala rename to tests/shared/src/test/scala/coursier/test/VersionIntervalTests.scala diff --git a/core/shared/src/test/scala/coursier/test/VersionTests.scala b/tests/shared/src/test/scala/coursier/test/VersionTests.scala similarity index 100% rename from core/shared/src/test/scala/coursier/test/VersionTests.scala rename to tests/shared/src/test/scala/coursier/test/VersionTests.scala diff --git a/core/shared/src/test/scala/coursier/test/package.scala b/tests/shared/src/test/scala/coursier/test/package.scala similarity index 100% rename from core/shared/src/test/scala/coursier/test/package.scala rename to tests/shared/src/test/scala/coursier/test/package.scala diff --git a/web/src/main/scala/coursier/web/Backend.scala b/web/src/main/scala/coursier/web/Backend.scala index 805cdbecc..56117eb1c 100644 --- a/web/src/main/scala/coursier/web/Backend.scala +++ b/web/src/main/scala/coursier/web/Backend.scala @@ -1,7 +1,8 @@ package coursier package web -import coursier.core.{ Repository, MavenRepository, MavenSource } +import coursier.maven.MavenSource + import japgolly.scalajs.react.vdom.{ TagMod, Attr } import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml import japgolly.scalajs.react.{ ReactEventI, ReactComponentB, BackendScope } @@ -10,6 +11,7 @@ import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue import org.scalajs.jquery.jQuery import scala.concurrent.Future +import scalaz.concurrent.Task import scala.scalajs.js import js.Dynamic.{ global => g } @@ -32,6 +34,22 @@ case class State( ) class Backend($: BackendScope[Unit, State]) { + + def fetch( + repositories: Seq[core.Repository], + fetch: Repository.Fetch[Task] + ): ResolutionProcess.Fetch[Task] = { + + modVers => Task.gatherUnordered( + modVers.map { case (module, version) => + Repository.find(repositories, module, version, fetch) + .run + .map((module, version) -> _) + } + ) + } + + def updateDepGraph(resolution: Resolution) = { println("Rendering canvas") @@ -138,7 +156,7 @@ class Backend($: BackendScope[Unit, State]) { g.$("#resLogTab a:last").tab("show") $.modState(_.copy(resolving = true, log = Nil)) - val logger: MavenRepository.Logger = new MavenRepository.Logger { + val logger: Platform.Logger = new Platform.Logger { def fetched(url: String) = { println(s"<- $url") $.modState(s => s.copy(log = s"<- $url" +: s.log)) @@ -163,11 +181,9 @@ class Backend($: BackendScope[Unit, State]) { ) ) - implicit val cachePolicy = CachePolicy.Default - res .process - .run(s.repositories.map(item => item._2.copy(logger = Some(logger))), 100) + .run(fetch(s.repositories.map { case (_, repo) => repo }, Platform.artifactWithLogger(logger)), 100) } // For reasons that are unclear to me, not delaying this when using the runNow execution context @@ -702,7 +718,7 @@ object App { val initialState = State( Nil, - Seq("central" -> Repository.mavenCentral), + Seq("central" -> MavenRepository("https://repo1.maven.org/maven2/")), ResolutionOptions(), None, -1,