Move things around

- core doesn't depend anymore on scalaz-concurrent, only on scalaz-core
- metadata and JARs downloading now unified (merged with the latter)
- ...

appveyor.yml

@@ -16,7 +16,7 @@ install:
 build_script:
   - sbt clean compile publish-local
 test_script:
-  - sbt coreJVM/test # Would node be around for coreJS/test?
+  - sbt testsJVM/test # Would node be around for testsJS/test?
 cache:
   - C:\sbt\
   - C:\Users\appveyor\.m2

build.sbt

@@ -65,41 +65,65 @@ lazy val commonSettings = baseCommonSettings ++ Seq(
   }
 )
 
+
 lazy val core = crossProject
   .settings(commonSettings: _*)
   .settings(publishingSettings: _*)
   .settings(
-    name := "coursier",
+    name := "coursier"
-    libraryDependencies += "org.scala-lang.modules" %% "scala-async" % "0.9.1" % "provided",
-    unmanagedResourceDirectories in Compile += (baseDirectory in LocalRootProject).value / "core" / "shared" / "src" / "main" / "resources",
-    unmanagedResourceDirectories in Test += (baseDirectory in LocalRootProject).value / "core" / "shared" / "src" / "test" / "resources",
-    testFrameworks += new TestFramework("utest.runner.Framework")
   )
   .jvmSettings(
-    libraryDependencies ++= Seq(
+    libraryDependencies ++=
-      "org.scalaz" %% "scalaz-concurrent" % "7.1.2",
+      Seq(
-      "com.lihaoyi" %% "utest" % "0.3.0" % "test"
+        "org.scalaz" %% "scalaz-core" % "7.1.2"
-    ) ++ {
+      ) ++ {
-      if (scalaVersion.value.startsWith("2.10.")) Seq()
+        if (scalaVersion.value.startsWith("2.10.")) Seq()
-      else Seq(
+        else Seq(
-        "org.scala-lang.modules" %% "scala-xml" % "1.0.3"
+          "org.scala-lang.modules" %% "scala-xml" % "1.0.3"
-      )
+        )
-    }
+      }
   )
   .jsSettings(
     libraryDependencies ++= Seq(
-      "org.scala-js" %%% "scalajs-dom" % "0.8.0",
       "com.github.japgolly.fork.scalaz" %%% "scalaz-core" % (if (scalaVersion.value.startsWith("2.10.")) "7.1.1" else "7.1.2"),
-      "be.doeraene" %%% "scalajs-jquery" % "0.8.0",
+      "org.scala-js" %%% "scalajs-dom" % "0.8.0",
-      "com.lihaoyi" %%% "utest" % "0.3.0" % "test"
+      "be.doeraene" %%% "scalajs-jquery" % "0.8.0"
-    ),
+    )
-    postLinkJSEnv := NodeJSEnv().value,
-    scalaJSStage in Global := FastOptStage
   )
 
 lazy val coreJvm = core.jvm
 lazy val coreJs = core.js
 
+lazy val `fetch-js` = project
+  .enablePlugins(ScalaJSPlugin)
+  .dependsOn(coreJs)
+  .settings(commonSettings)
+  .settings(noPublishSettings)
+  .settings(
+    name := "coursier-fetch-js"
+  )
+
+lazy val tests = crossProject
+  .dependsOn(core)
+  .settings(commonSettings: _*)
+  .settings(noPublishSettings: _*)
+  .settings(
+    name := "coursier-tests",
+    libraryDependencies ++= Seq(
+      "org.scala-lang.modules" %% "scala-async" % "0.9.1" % "provided",
+      "com.lihaoyi" %%% "utest" % "0.3.0" % "test"
+    ),
+    unmanagedResourceDirectories in Test += (baseDirectory in LocalRootProject).value / "tests" / "shared" / "src" / "test" / "resources",
+    testFrameworks += new TestFramework("utest.runner.Framework")
+  )
+  .jsSettings(
+    postLinkJSEnv := NodeJSEnv().value,
+    scalaJSStage in Global := FastOptStage
+  )
+
+lazy val testsJvm = tests.jvm.dependsOn(files % "test")
+lazy val testsJs = tests.js.dependsOn(`fetch-js` % "test")
+
 lazy val files = project
   .dependsOn(coreJvm)
   .settings(commonSettings)
@@ -107,21 +131,33 @@ lazy val files = project
   .settings(
     name := "coursier-files",
     libraryDependencies ++= Seq(
-      "com.lihaoyi" %% "utest" % "0.3.0" % "test"
+      "org.scalaz" %% "scalaz-concurrent" % "7.1.2"
-    ),
+    )
-    testFrameworks += new TestFramework("utest.runner.Framework")
+  )
+
+lazy val bootstrap = project
+  .settings(baseCommonSettings)
+  .settings(publishingSettings)
+  .settings(
+    name := "coursier-bootstrap",
+    artifactName := {
+      val artifactName0 = artifactName.value
+      (sv, m, artifact) =>
+        if (artifact.`type` == "jar" && artifact.extension == "jar")
+          "bootstrap.jar"
+        else
+          artifactName0(sv, m, artifact)
+    },
+    crossPaths := false,
+    autoScalaLibrary := false,
+    javacOptions in doc := Seq()
   )
 
 lazy val cli = project
   .dependsOn(coreJvm, files)
   .settings(commonSettings)
   .settings(publishingSettings)
-  .settings(packAutoSettings ++ publishPackTxzArchive ++ publishPackZipArchive)
+  .settings(packAutoSettings)
-  .settings(
-    packArchivePrefix := s"coursier-cli_${scalaBinaryVersion.value}",
-    packArchiveTxzArtifact := Artifact("coursier-cli", "arch", "tar.xz"),
-    packArchiveZipArtifact := Artifact("coursier-cli", "arch", "zip")
-  )
   .settings(
     name := "coursier-cli",
     libraryDependencies ++= Seq(
@@ -135,7 +171,7 @@ lazy val cli = project
 
 lazy val web = project
   .enablePlugins(ScalaJSPlugin)
-  .dependsOn(coreJs)
+  .dependsOn(coreJs, `fetch-js`)
   .settings(commonSettings)
   .settings(noPublishSettings)
   .settings(
@@ -164,29 +200,7 @@ lazy val web = project
     )
   )
 
-lazy val bootstrap = project
-  .settings(baseCommonSettings)
-  .settings(publishingSettings)
-  .settings(
-    name := "coursier-bootstrap",
-    artifactName := {
-      val artifactName0 = artifactName.value
-      (sv, m, artifact) =>
-        if (artifact.`type` == "jar" && artifact.extension == "jar")
-          "bootstrap.jar"
-        else
-          artifactName0(sv, m, artifact)
-    },
-    crossPaths := false,
-    autoScalaLibrary := false,
-    javacOptions in doc := Seq()
-  )
-
 lazy val `coursier` = project.in(file("."))
-  .aggregate(coreJvm, coreJs, files, cli, web, bootstrap)
+  .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, files, bootstrap, cli, web)
   .settings(commonSettings)
   .settings(noPublishSettings)
-  .settings(
-    (unmanagedSourceDirectories in Compile) := Nil,
-    (unmanagedSourceDirectories in Test) := Nil
-  )

cli/src/main/scala/coursier/cli/Coursier.scala

@@ -326,13 +326,7 @@ case class Bootstrap(
         sys.exit(1)
     }
 
-  // scala-library version in the resulting JARs has to match the one in the bootstrap JAR
+  val helper = new Helper(common, remainingArgs)
-  // This should be enforced more strictly (possibly by having one bootstrap JAR per scala version).
-
-  val helper = new Helper(
-    common,
-    remainingArgs :+ s"org.scala-lang:scala-library:${scala.util.Properties.versionNumberString}"
-  )
 
   val artifacts = helper.res.artifacts
 

cli/src/main/scala/coursier/cli/Helper.scala

@@ -1,12 +1,9 @@
-package coursier.cli
+package coursier
+package cli
 
 import java.io.File
 import java.util.UUID
 
-import caseapp.CaseApp
-import coursier._
-import coursier.core.{ CachePolicy, MavenRepository }
-
 import scalaz.{ \/-, -\/ }
 import scalaz.concurrent.Task
 
@@ -31,16 +28,8 @@ object Helper {
 
   def errPrintln(s: String) = Console.err.println(s)
 
-  def defaultLogger: MavenRepository.Logger with Files.Logger =
+  def defaultLogger: Files.Logger =
-    new MavenRepository.Logger with Files.Logger {
+    new Files.Logger {
-      def downloading(url: String) =
-        errPrintln(s"Downloading $url")
-      def downloaded(url: String, success: Boolean) =
-        if (!success)
-          errPrintln(s"Failed: $url")
-      def readingFromCache(f: File) = {}
-      def puttingInCache(f: File) = {}
-
       def foundLocally(f: File) = {}
       def downloadingArtifact(url: String) =
         errPrintln(s"Downloading $url")
@@ -49,21 +38,8 @@ object Helper {
           errPrintln(s"Failed: $url")
     }
 
-  def verboseLogger: MavenRepository.Logger with Files.Logger =
+  def verboseLogger: Files.Logger =
-    new MavenRepository.Logger with Files.Logger {
+    new Files.Logger {
-      def downloading(url: String) =
-        errPrintln(s"Downloading $url")
-      def downloaded(url: String, success: Boolean) =
-        errPrintln(
-          if (success) s"Downloaded $url"
-          else s"Failed: $url"
-        )
-      def readingFromCache(f: File) = {
-        errPrintln(s"Reading ${fileRepr(f)} from cache")
-      }
-      def puttingInCache(f: File) =
-        errPrintln(s"Writing ${fileRepr(f)} in cache")
-
       def foundLocally(f: File) =
         errPrintln(s"Found locally ${fileRepr(f)}")
       def downloadingArtifact(url: String) =
@@ -176,13 +152,18 @@ class Helper(
     sys.exit(1)
   }
 
-  val (repositories0, fileCaches) = repositoryIdsOpt0
+  val files = {
+    var files0 = cache
+      .files()
+      .copy(logger = logger)
+    files0 = files0.copy(concurrentDownloadCount = parallel)
+    files0
+  }
+
+  val (repositories, fileCaches) = repositoryIdsOpt0
     .collect { case Right(v) => v }
     .unzip
 
-  val repositories = repositories0
-    .map(_.copy(logger = logger))
-
   val (rawDependencies, extraArgs) = {
     val idxOpt = Some(remainingArgs.indexOf("--")).filter(_ >= 0)
     idxOpt.fold((remainingArgs, Seq.empty[String])) { idx =>
@@ -221,7 +202,7 @@ class Helper(
     filter = Some(dep => keepOptional || !dep.optional)
   )
 
-  val fetchQuiet = coursier.fetchLocalFirst(repositories)
+  val fetchQuiet = coursier.Fetch(repositories, files.fetch)
   val fetch0 =
     if (verbose0 == 0) fetchQuiet
     else {
@@ -317,14 +298,6 @@ class Helper(
       l
     }
 
-    val files = {
-      var files0 = cache
-        .files()
-        .copy(logger = logger)
-      files0 = files0.copy(concurrentDownloadCount = parallel)
-      files0
-    }
-
     val tasks = artifacts.map(artifact => files.file(artifact).run.map(artifact.->))
     def printTask = Task{
       if (verbose0 >= 0 && artifacts.nonEmpty)

core/js/src/main/scala/coursier/core/compatibility/package.scala

@@ -4,6 +4,8 @@ import scala.scalajs.js
 import js.Dynamic.{ global => g }
 import org.scalajs.dom.raw.NodeList
 
+import coursier.util.Xml
+
 package object compatibility {
   def option[A](a: js.Dynamic): Option[A] =
     if (js.isUndefined(a)) None

core/jvm/src/main/scala/coursier/core/MavenRepository.scala

@@ -1,135 +0,0 @@
-package coursier
-package core
-
-import java.io._
-import java.net.{ URI, URL }
-
-import scala.io.Codec
-import scalaz._, Scalaz._
-import scalaz.concurrent.Task
-
-case class MavenRepository(
-  root: String,
-  cache: Option[File] = None,
-  ivyLike: Boolean = false,
-  logger: Option[MavenRepository.Logger] = None
-) extends BaseMavenRepository(root, ivyLike) {
-
-  val isLocal = root.startsWith("file:/")
-
-  def fetch(
-    artifact: Artifact,
-    cachePolicy: CachePolicy
-  ): EitherT[Task, String, String] = {
-
-    def locally(eitherFile: String \/ File) = {
-      Task {
-        for {
-          f0 <- eitherFile
-          f <- Some(f0).filter(_.exists()).toRightDisjunction("Not found in cache")
-          content <- \/.fromTryCatchNonFatal{
-            logger.foreach(_.readingFromCache(f))
-            scala.io.Source.fromFile(f)(Codec.UTF8).mkString
-          }.leftMap(_.getMessage)
-        } yield content
-      }
-    }
-
-    if (isLocal) EitherT(locally(\/-(new File(new URI(root + artifact.url) .getPath))))
-    else {
-      lazy val localFile = {
-        for {
-          cache0 <- cache.toRightDisjunction("No cache")
-          f = new File(cache0, artifact.url)
-        } yield f
-      }
-
-      def remote = {
-        val urlStr = root + artifact.url
-        val url = new URL(urlStr)
-
-        def log = Task(logger.foreach(_.downloading(urlStr)))
-        def get = {
-          val conn = url.openConnection()
-          // Dummy user-agent instead of the default "Java/...",
-          // so that we are not returned incomplete/erroneous metadata
-          // (Maven 2 compatibility? - happens for snapshot versioning metadata,
-          // this is SO FUCKING CRAZY)
-          conn.setRequestProperty("User-Agent", "")
-          MavenRepository.readFully(conn.getInputStream())
-        }
-        def logEnd(success: Boolean) = logger.foreach(_.downloaded(urlStr, success))
-
-        log
-          .flatMap(_ => get)
-          .map{ res => logEnd(res.isRight); res }
-      }
-
-      def save(s: String) = {
-        localFile.fold(_ => Task.now(()), f =>
-          Task {
-            if (!f.exists()) {
-              logger.foreach(_.puttingInCache(f))
-              f.getParentFile.mkdirs()
-              val w = new PrintWriter(f)
-              try w.write(s)
-              finally w.close()
-              ()
-            }
-          }
-        )
-      }
-
-      EitherT(
-        cachePolicy[String \/ String](
-          _.isLeft )(
-          locally(localFile) )(
-          _ => CachePolicy.saving(remote)(save)
-        )
-      )
-    }
-  }
-
-}
-
-object MavenRepository {
-
-  trait Logger {
-    def downloading(url: String): Unit
-    def downloaded(url: String, success: Boolean): Unit
-    def readingFromCache(f: File): Unit
-    def puttingInCache(f: File): Unit
-  }
-
-  def readFullySync(is: InputStream) = {
-    val buffer = new ByteArrayOutputStream()
-    val data = Array.ofDim[Byte](16384)
-
-    var nRead = is.read(data, 0, data.length)
-    while (nRead != -1) {
-      buffer.write(data, 0, nRead)
-      nRead = is.read(data, 0, data.length)
-    }
-
-    buffer.flush()
-    buffer.toByteArray
-  }
-
-  def readFully(is: => InputStream) =
-    Task {
-      \/.fromTryCatchNonFatal {
-        val is0 = is
-        val b =
-          try readFullySync(is0)
-          finally is0.close()
-
-        new String(b, "UTF-8")
-      } .leftMap{
-        case e: java.io.FileNotFoundException =>
-          s"Not found: ${e.getMessage}"
-        case e =>
-          s"$e: ${e.getMessage}"
-      }
-    }
-
-}

core/jvm/src/main/scala/coursier/core/compatibility/package.scala

@@ -1,5 +1,7 @@
 package coursier.core
 
+import coursier.util.Xml
+
 package object compatibility {
 
   implicit class RichChar(val c: Char) extends AnyVal {
@@ -27,4 +29,5 @@ package object compatibility {
 
   def encodeURIComponent(s: String): String =
     new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString
+
 }

core/shared/src/main/scala/coursier/core/Repository.scala

@@ -1,30 +1,24 @@
 package coursier.core
 
-import scalaz.{ -\/, \/-, \/, EitherT }
+import scala.language.higherKinds
-import scalaz.concurrent.Task
 
-import java.io.File
+import scalaz._
 
 import coursier.core.compatibility.encodeURIComponent
 
 trait Repository {
-  def find(
+  def find[F[_]](
     module: Module,
-    version: String
+    version: String,
+    fetch: Repository.Fetch[F]
   )(implicit
-    cachePolicy: CachePolicy
+    F: Monad[F]
-  ): EitherT[Task, String, (Artifact.Source, Project)]
+  ): EitherT[F, String, (Artifact.Source, Project)]
 }
 
 object Repository {
 
-  val mavenCentral = MavenRepository("https://repo1.maven.org/maven2/")
+  type Fetch[F[_]] = Artifact => EitherT[F, String, String]
-
-  val sonatypeReleases = MavenRepository("https://oss.sonatype.org/content/repositories/releases/")
-  val sonatypeSnapshots = MavenRepository("https://oss.sonatype.org/content/repositories/snapshots/")
-
-  lazy val ivy2Local = MavenRepository(new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString, ivyLike = true)
-
 
   /**
    * Try to find `module` among `repositories`.
@@ -37,38 +31,33 @@ object Repository {
    * version (e.g. version interval). Which version get chosen depends on
    * the repository implementation.
    */
-  def find(
+  def find[F[_]](
     repositories: Seq[Repository],
     module: Module,
-    version: String
+    version: String,
+    fetch: Repository.Fetch[F]
   )(implicit
-    cachePolicy: CachePolicy
+    F: Monad[F]
-  ): EitherT[Task, Seq[String], (Artifact.Source, Project)] = {
+  ): EitherT[F, Seq[String], (Artifact.Source, Project)] = {
 
     val lookups = repositories
-      .map(repo => repo -> repo.find(module, version).run)
+      .map(repo => repo -> repo.find(module, version, fetch).run)
 
-    val task = lookups
+    val task = lookups.foldLeft[F[Seq[String] \/ (Artifact.Source, Project)]](F.point(-\/(Nil))) {
-      .foldLeft(Task.now(-\/(Nil)): Task[Seq[String] \/ (Artifact.Source, Project)]) {
+      case (acc, (repo, eitherProjTask)) =>
-        case (acc, (repo, eitherProjTask)) =>
+        F.bind(acc) {
-          acc
+          case -\/(errors) =>
-            .flatMap {
+            F.map(eitherProjTask)(_.flatMap{case (source, project) =>
-              case -\/(errors) =>
+              if (project.module == module) \/-((source, project))
-                eitherProjTask
+              else -\/(s"Wrong module returned (expected: $module, got: ${project.module})")
-                  .map(res => res
+            }.leftMap(error => error +: errors))
-                    .flatMap{case (source, project) =>
-                      if (project.module == module) \/-((source, project))
-                      else -\/(s"Wrong module returned (expected: $module, got: ${project.module})")
-                    }
-                    .leftMap(error => error +: errors)
-                  )
 
-              case res @ \/-(_) =>
+          case res @ \/-(_) =>
-                Task.now(res)
+            F.point(res)
-            }
+        }
-      }
+    }
 
-    EitherT(task.map(_.leftMap(_.reverse)))
+    EitherT(F.map(task)(_.leftMap(_.reverse)))
       .map {case x @ (_, proj) =>
         assert(proj.module == module)
         x
@@ -101,401 +90,3 @@ object Repository {
   }
 }
 
-case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
-  import Repository._
-  import BaseMavenRepository._
-
-  def artifacts(
-    dependency: Dependency,
-    project: Project
-  ): Seq[Artifact] = {
-
-    def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) =
-      ivyLikePath(
-        dependency.module.organization,
-        dependency.module.name,
-        project.version,
-        subDir,
-        baseSuffix,
-        ext
-      )
-
-    val path =
-      if (ivyLike)
-        ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`)
-      else {
-        val versioning =
-          project
-            .snapshotVersioning
-            .flatMap(versioning =>
-              mavenVersioning(versioning, dependency.attributes.classifier, dependency.attributes.`type`)
-            )
-
-        dependency.module.organization.split('.').toSeq ++ Seq(
-          dependency.module.name,
-          project.version,
-          s"${dependency.module.name}-${versioning getOrElse project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}"
-        )
-      }
-
-    var artifact =
-      Artifact(
-        root + path.mkString("/"),
-        Map.empty,
-        Map.empty,
-        dependency.attributes
-      )
-      .withDefaultChecksums
-
-    if (dependency.attributes.`type` == "jar") {
-      artifact = artifact.withDefaultSignature
-
-      // FIXME Snapshot versioning of sources and javadoc is not taken into account here.
-      // Will be ok if it's the same as the main JAR though.
-
-      artifact =
-        if (ivyLike) {
-          val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/")
-          val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/")
-
-          artifact
-            .copy(
-              extra = artifact.extra ++ Map(
-                "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
-                  .withDefaultChecksums
-                  .withDefaultSignature,
-                "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
-                  .withDefaultChecksums
-                  .withDefaultSignature
-            ))
-        } else
-          artifact
-            .withJavadocSources
-    }
-
-    Seq(artifact)
-  }
-}
-
-object BaseMavenRepository {
-
-  def ivyLikePath(
-    org: String,
-    name: String,
-    version: String,
-    subDir: String,
-    baseSuffix: String,
-    ext: String
-  ) =
-    Seq(
-      org,
-      name,
-      version,
-      subDir,
-      s"$name$baseSuffix.$ext"
-    )
-
-  def mavenVersioning(
-    snapshotVersioning: SnapshotVersioning,
-    classifier: String,
-    extension: String
-  ): Option[String] =
-    snapshotVersioning
-      .snapshotVersions
-      .find(v => v.classifier == classifier && v.extension == extension)
-      .map(_.value)
-      .filter(_.nonEmpty)
-
-}
-
-abstract class BaseMavenRepository(
-  root: String,
-  ivyLike: Boolean
-) extends Repository {
-
-  def fetch(
-    artifact: Artifact,
-    cachePolicy: CachePolicy
-  ): EitherT[Task, String, String]
-
-  import Repository._
-  import BaseMavenRepository._
-
-  val source = MavenSource(root, ivyLike)
-
-  def projectArtifact(
-    module: Module,
-    version: String,
-    versioningValue: Option[String]
-  ): Artifact = {
-
-    val path = (
-      if (ivyLike)
-        ivyLikePath(
-          module.organization,
-          module.name,
-          versioningValue getOrElse version,
-          "poms",
-          "",
-          "pom"
-        )
-      else
-        module.organization.split('.').toSeq ++ Seq(
-          module.name,
-          version,
-          s"${module.name}-${versioningValue getOrElse version}.pom"
-        )
-    ) .map(encodeURIComponent)
-
-    Artifact(
-      path.mkString("/"),
-      Map.empty,
-      Map.empty,
-      Attributes("pom", "")
-    )
-    .withDefaultSignature
-  }
-
-  def versionsArtifact(module: Module): Option[Artifact] =
-    if (ivyLike) None
-    else {
-      val path = (
-        module.organization.split('.').toSeq ++ Seq(
-          module.name,
-          "maven-metadata.xml"
-        )
-      ) .map(encodeURIComponent)
-
-      val artifact =
-        Artifact(
-          path.mkString("/"),
-          Map.empty,
-          Map.empty,
-          Attributes("pom", "")
-        )
-        .withDefaultChecksums
-
-      Some(artifact)
-    }
-
-  def snapshotVersioningArtifact(
-    module: Module,
-    version: String
-  ): Option[Artifact] =
-    if (ivyLike) None
-    else {
-      val path = (
-        module.organization.split('.').toSeq ++ Seq(
-          module.name,
-          version,
-          "maven-metadata.xml"
-        )
-      ) .map(encodeURIComponent)
-
-      val artifact =
-        Artifact(
-          path.mkString("/"),
-          Map.empty,
-          Map.empty,
-          Attributes("pom", "")
-        )
-        .withDefaultChecksums
-
-      Some(artifact)
-    }
-
-  def versions(
-    module: Module,
-    cachePolicy: CachePolicy = CachePolicy.Default
-  ): EitherT[Task, String, Versions] = {
-
-    EitherT(
-      versionsArtifact(module) match {
-        case None => Task.now(-\/("Not supported"))
-        case Some(artifact) =>
-          fetch(artifact, cachePolicy)
-            .run
-            .map(eitherStr =>
-              for {
-                str <- eitherStr
-                xml <- \/.fromEither(compatibility.xmlParse(str))
-                _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found")
-                versions <- Xml.versions(xml)
-              } yield versions
-            )
-      }
-    )
-  }
-
-  def snapshotVersioning(
-    module: Module,
-    version: String,
-    cachePolicy: CachePolicy = CachePolicy.Default
-  ): EitherT[Task, String, SnapshotVersioning] = {
-
-    EitherT(
-      snapshotVersioningArtifact(module, version) match {
-        case None => Task.now(-\/("Not supported"))
-        case Some(artifact) =>
-          fetch(artifact, cachePolicy)
-            .run
-            .map(eitherStr =>
-              for {
-                str <- eitherStr
-                xml <- \/.fromEither(compatibility.xmlParse(str))
-                _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found")
-                snapshotVersioning <- Xml.snapshotVersioning(xml)
-              } yield snapshotVersioning
-            )
-      }
-    )
-  }
-
-  def findNoInterval(
-    module: Module,
-    version: String,
-    cachePolicy: CachePolicy
-  ): EitherT[Task, String, Project] =
-    EitherT{
-      def withSnapshotVersioning =
-        snapshotVersioning(module, version, cachePolicy)
-          .flatMap { snapshotVersioning =>
-            val versioningOption =
-              mavenVersioning(snapshotVersioning, "", "jar")
-                .orElse(mavenVersioning(snapshotVersioning, "", ""))
-
-            versioningOption match {
-              case None =>
-                EitherT[Task, String, Project](
-                  Task.now(-\/("No snapshot versioning value found"))
-                )
-              case versioning @ Some(_) =>
-                findVersioning(module, version, versioning, cachePolicy)
-                  .map(_.copy(snapshotVersioning = Some(snapshotVersioning)))
-            }
-          }
-
-      findVersioning(module, version, None, cachePolicy)
-        .run
-        .flatMap{ eitherProj =>
-          if (eitherProj.isLeft)
-            withSnapshotVersioning
-              .run
-              .map(eitherProj0 =>
-                if (eitherProj0.isLeft)
-                  eitherProj
-                else
-                  eitherProj0
-              )
-          else
-            Task.now(eitherProj)
-        }
-    }
-
-  def findVersioning(
-    module: Module,
-    version: String,
-    versioningValue: Option[String],
-    cachePolicy: CachePolicy
-  ): EitherT[Task, String, Project] = {
-
-    EitherT {
-      fetch(projectArtifact(module, version, versioningValue), cachePolicy)
-        .run
-        .map(eitherStr =>
-          for {
-            str <- eitherStr
-            xml <- \/.fromEither(compatibility.xmlParse(str))
-            _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found")
-            proj <- Xml.project(xml)
-          } yield proj
-        )
-    }
-  }
-
-  def find(
-    module: Module,
-    version: String
-  )(implicit
-    cachePolicy: CachePolicy
-  ): EitherT[Task, String, (Artifact.Source, Project)] = {
-
-    Parse.versionInterval(version)
-      .filter(_.isValid) match {
-        case None =>
-          findNoInterval(module, version, cachePolicy).map((source, _))
-        case Some(itv) =>
-          versions(module, cachePolicy)
-            .flatMap { versions0 =>
-              val eitherVersion = {
-                val release = Version(versions0.release)
-
-                if (itv.contains(release)) \/-(versions0.release)
-                else {
-                  val inInterval = versions0.available
-                    .map(Version(_))
-                    .filter(itv.contains)
-
-                  if (inInterval.isEmpty) -\/(s"No version found for $version")
-                  else \/-(inInterval.max.repr)
-                }
-              }
-
-              eitherVersion match {
-                case -\/(reason) => EitherT[Task, String, (Artifact.Source, Project)](Task.now(-\/(reason)))
-                case \/-(version0) =>
-                  findNoInterval(module, version0, cachePolicy)
-                    .map(_.copy(versions = Some(versions0)))
-                    .map((source, _))
-              }
-            }
-    }
-  }
-
-}
-
-sealed trait CachePolicy {
-  def apply[T](
-    tryRemote: T => Boolean )(
-    local: => Task[T] )(
-    remote: Option[T] => Task[T]
-  ): Task[T]
-}
-
-object CachePolicy {
-  def saving[E,T](
-    remote: => Task[E \/ T] )(
-    save: T => Task[Unit]
-  ): Task[E \/ T] = {
-    for {
-      res <- remote
-      _ <- res.fold(_ => Task.now(()), t => save(t))
-    } yield res
-  }
-
-  case object Default extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      local
-        .flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res))
-  }
-  case object LocalOnly extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      local
-  }
-  case object ForceDownload extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      remote(None)
-  }
-}

core/shared/src/main/scala/coursier/maven/MavenRepository.scala

@@ -0,0 +1,279 @@
+package coursier.maven
+
+import coursier.core._
+import coursier.core.compatibility.encodeURIComponent
+
+import scala.language.higherKinds
+import scalaz._
+
+object MavenRepository {
+
+  def ivyLikePath(
+    org: String,
+    name: String,
+    version: String,
+    subDir: String,
+    baseSuffix: String,
+    ext: String
+  ) =
+    Seq(
+      org,
+      name,
+      version,
+      subDir,
+      s"$name$baseSuffix.$ext"
+    )
+
+  def mavenVersioning(
+    snapshotVersioning: SnapshotVersioning,
+    classifier: String,
+    extension: String
+  ): Option[String] =
+    snapshotVersioning
+      .snapshotVersions
+      .find(v => v.classifier == classifier && v.extension == extension)
+      .map(_.value)
+      .filter(_.nonEmpty)
+
+}
+
+case class MavenRepository(
+  root: String,
+  ivyLike: Boolean = false
+) extends Repository {
+
+  import Repository._
+  import MavenRepository._
+
+  val root0 = if (root.endsWith("/")) root else root + "/"
+  val source = MavenSource(root0, ivyLike)
+
+  def projectArtifact(
+    module: Module,
+    version: String,
+    versioningValue: Option[String]
+  ): Artifact = {
+
+    val path = (
+      if (ivyLike)
+        ivyLikePath(
+          module.organization,
+          module.name,
+          versioningValue getOrElse version,
+          "poms",
+          "",
+          "pom"
+        )
+      else
+        module.organization.split('.').toSeq ++ Seq(
+          module.name,
+          version,
+          s"${module.name}-${versioningValue getOrElse version}.pom"
+        )
+    ) .map(encodeURIComponent)
+
+    Artifact(
+      root0 + path.mkString("/"),
+      Map.empty,
+      Map.empty,
+      Attributes("pom", "")
+    )
+    .withDefaultChecksums
+    .withDefaultSignature
+  }
+
+  def versionsArtifact(module: Module): Option[Artifact] =
+    if (ivyLike) None
+    else {
+      val path = (
+        module.organization.split('.').toSeq ++ Seq(
+          module.name,
+          "maven-metadata.xml"
+        )
+      ) .map(encodeURIComponent)
+
+      val artifact =
+        Artifact(
+          root0 + path.mkString("/"),
+          Map.empty,
+          Map.empty,
+          Attributes("pom", "")
+        )
+        .withDefaultChecksums
+        .withDefaultChecksums
+
+      Some(artifact)
+    }
+
+  def snapshotVersioningArtifact(
+    module: Module,
+    version: String
+  ): Option[Artifact] =
+    if (ivyLike) None
+    else {
+      val path = (
+        module.organization.split('.').toSeq ++ Seq(
+          module.name,
+          version,
+          "maven-metadata.xml"
+        )
+      ) .map(encodeURIComponent)
+
+      val artifact =
+        Artifact(
+          root0 + path.mkString("/"),
+          Map.empty,
+          Map.empty,
+          Attributes("pom", "")
+        )
+        .withDefaultChecksums
+        .withDefaultSignature
+
+      Some(artifact)
+    }
+
+  def versions[F[_]](
+    module: Module,
+    fetch: Repository.Fetch[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, Versions] =
+    EitherT(
+      versionsArtifact(module) match {
+        case None => F.point(-\/("Not supported"))
+        case Some(artifact) =>
+          F.map(fetch(artifact).run)(eitherStr =>
+            for {
+              str <- eitherStr
+              xml <- \/.fromEither(compatibility.xmlParse(str))
+              _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found")
+              versions <- Pom.versions(xml)
+            } yield versions
+          )
+      }
+    )
+
+  def snapshotVersioning[F[_]](
+    module: Module,
+    version: String,
+    fetch: Repository.Fetch[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, SnapshotVersioning] = {
+
+    EitherT(
+      snapshotVersioningArtifact(module, version) match {
+        case None => F.point(-\/("Not supported"))
+        case Some(artifact) =>
+          F.map(fetch(artifact).run)(eitherStr =>
+            for {
+              str <- eitherStr
+              xml <- \/.fromEither(compatibility.xmlParse(str))
+              _ <- if (xml.label == "metadata") \/-(()) else -\/("Metadata not found")
+              snapshotVersioning <- Pom.snapshotVersioning(xml)
+            } yield snapshotVersioning
+          )
+      }
+    )
+  }
+
+  def findNoInterval[F[_]](
+    module: Module,
+    version: String,
+    fetch: Repository.Fetch[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, Project] =
+    EitherT {
+      def withSnapshotVersioning =
+        snapshotVersioning(module, version, fetch).flatMap { snapshotVersioning =>
+          val versioningOption =
+            mavenVersioning(snapshotVersioning, "", "jar")
+              .orElse(mavenVersioning(snapshotVersioning, "", ""))
+
+          versioningOption match {
+            case None =>
+              EitherT[F, String, Project](
+                F.point(-\/("No snapshot versioning value found"))
+              )
+            case versioning @ Some(_) =>
+              findVersioning(module, version, versioning, fetch)
+                .map(_.copy(snapshotVersioning = Some(snapshotVersioning)))
+          }
+        }
+
+      F.bind(findVersioning(module, version, None, fetch).run) { eitherProj =>
+        if (eitherProj.isLeft)
+          F.map(withSnapshotVersioning.run)(eitherProj0 =>
+            if (eitherProj0.isLeft)
+              eitherProj
+            else
+              eitherProj0
+          )
+        else
+          F.point(eitherProj)
+      }
+    }
+
+  def findVersioning[F[_]](
+    module: Module,
+    version: String,
+    versioningValue: Option[String],
+    fetch: Repository.Fetch[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, Project] = {
+
+    fetch(projectArtifact(module, version, versioningValue)).flatMap { str =>
+      EitherT {
+        F.point {
+          (for {
+            xml <- \/.fromEither(compatibility.xmlParse(str))
+            _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found")
+            proj <- Pom.project(xml)
+          } yield proj): (String \/ Project)
+        }
+      }
+    }
+  }
+
+  def find[F[_]](
+    module: Module,
+    version: String,
+    fetch: Repository.Fetch[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, (Artifact.Source, Project)] = {
+
+    Parse.versionInterval(version)
+      .filter(_.isValid) match {
+        case None =>
+          findNoInterval(module, version, fetch).map((source, _))
+        case Some(itv) =>
+          versions(module, fetch).flatMap { versions0 =>
+            val eitherVersion = {
+              val release = Version(versions0.release)
+
+              if (itv.contains(release)) \/-(versions0.release)
+              else {
+                val inInterval = versions0.available
+                  .map(Version(_))
+                  .filter(itv.contains)
+
+                if (inInterval.isEmpty) -\/(s"No version found for $version")
+                else \/-(inInterval.max.repr)
+              }
+            }
+
+            eitherVersion match {
+              case -\/(reason) => EitherT[F, String, (Artifact.Source, Project)](F.point(-\/(reason)))
+              case \/-(version0) =>
+                findNoInterval(module, version0, fetch)
+                  .map(_.copy(versions = Some(versions0)))
+                  .map((source, _))
+            }
+          }
+    }
+  }
+
+}

core/shared/src/main/scala/coursier/maven/MavenSource.scala

@@ -0,0 +1,79 @@
+package coursier.maven
+
+import coursier.core._
+
+case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
+  import Repository._
+  import MavenRepository._
+
+  def artifacts(
+    dependency: Dependency,
+    project: Project
+  ): Seq[Artifact] = {
+
+    def ivyLikePath0(subDir: String, baseSuffix: String, ext: String) =
+      ivyLikePath(
+        dependency.module.organization,
+        dependency.module.name,
+        project.version,
+        subDir,
+        baseSuffix,
+        ext
+      )
+
+    val path =
+      if (ivyLike)
+        ivyLikePath0(dependency.attributes.`type` + "s", "", dependency.attributes.`type`)
+      else {
+        val versioning =
+          project
+            .snapshotVersioning
+            .flatMap(versioning =>
+              mavenVersioning(versioning, dependency.attributes.classifier, dependency.attributes.`type`)
+            )
+
+        dependency.module.organization.split('.').toSeq ++ Seq(
+          dependency.module.name,
+          project.version,
+          s"${dependency.module.name}-${versioning getOrElse project.version}${Some(dependency.attributes.classifier).filter(_.nonEmpty).map("-"+_).mkString}.${dependency.attributes.`type`}"
+        )
+      }
+
+    var artifact =
+      Artifact(
+        root + path.mkString("/"),
+        Map.empty,
+        Map.empty,
+        dependency.attributes
+      )
+      .withDefaultChecksums
+
+    if (dependency.attributes.`type` == "jar") {
+      artifact = artifact.withDefaultSignature
+
+      // FIXME Snapshot versioning of sources and javadoc is not taken into account here.
+      // Will be ok if it's the same as the main JAR though.
+
+      artifact =
+        if (ivyLike) {
+          val srcPath = root + ivyLikePath0("srcs", "-sources", "jar").mkString("/")
+          val javadocPath = root + ivyLikePath0("docs", "-javadoc", "jar").mkString("/")
+
+          artifact
+            .copy(
+              extra = artifact.extra ++ Map(
+                "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
+                  .withDefaultChecksums
+                  .withDefaultSignature,
+                "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
+                  .withDefaultChecksums
+                  .withDefaultSignature
+            ))
+        } else
+          artifact
+            .withJavadocSources
+    }
+
+    Seq(artifact)
+  }
+}

core/shared/src/main/scala/coursier/maven/Pom.scala

@@ -1,28 +1,11 @@
-package coursier.core
+package coursier.maven
+
+import coursier.core._
 
 import scalaz._
 
-object Xml {
+object Pom {
-
+  import coursier.util.Xml._
-  /** A representation of an XML node/document, with different implementations on the JVM and JS */
-  trait Node {
-    def label: String
-    def child: Seq[Node]
-    def isText: Boolean
-    def textContent: String
-    def isElement: Boolean
-  }
-
-  object Node {
-    val empty: Node =
-      new Node {
-        val isText = false
-        val isElement = false
-        val child = Nil
-        val label = ""
-        val textContent = ""
-      }
-  }
 
   object Text {
     def unapply(n: Node): Option[String] =
@@ -340,7 +323,7 @@ object Xml {
         .traverseU(snapshotVersion)
     } yield SnapshotVersioning(
       Module(organization, name),
-      version, 
+      version,
       latest,
       release,
       timestamp,
@@ -350,5 +333,4 @@ object Xml {
       snapshotVersions
     )
   }
-
 }

core/shared/src/main/scala/coursier/package.scala

@@ -1,8 +1,6 @@
-import scalaz.{ -\/, \/- }
-import scalaz.concurrent.Task
 
 /**
- * Pulls definitions from coursier.core, sometimes with default arguments.
+ * Mainly pulls definitions from coursier.core, sometimes with default arguments.
  */
 package object coursier {
 
@@ -37,10 +35,10 @@ package object coursier {
   }
 
   type Project = core.Project
-  val Project: core.Project.type = core.Project
+  val Project = core.Project
 
   type Profile = core.Profile
-  val Profile: core.Profile.type = core.Profile
+  val Profile = core.Profile
 
   type Module = core.Module
   object Module {
@@ -51,13 +49,13 @@ package object coursier {
   type ModuleVersion = (core.Module, String)
 
   type Scope = core.Scope
-  val Scope: core.Scope.type = core.Scope
+  val Scope = core.Scope
-
-  type CachePolicy = core.CachePolicy
-  val CachePolicy: core.CachePolicy.type = core.CachePolicy
 
   type Repository = core.Repository
-  val Repository: core.Repository.type = core.Repository
+  val Repository = core.Repository
+
+  type MavenRepository = maven.MavenRepository
+  val MavenRepository = maven.MavenRepository
 
   type Resolution = core.Resolution
   object Resolution {
@@ -83,54 +81,14 @@ package object coursier {
   }
 
   type Artifact = core.Artifact
-  val Artifact: core.Artifact.type = core.Artifact
+  val Artifact = core.Artifact
 
   type ResolutionProcess = core.ResolutionProcess
-  val ResolutionProcess: core.ResolutionProcess.type = core.ResolutionProcess
+  val ResolutionProcess = core.ResolutionProcess
 
   implicit class ResolutionExtensions(val underlying: Resolution) extends AnyVal {
 
     def process: ResolutionProcess = ResolutionProcess(underlying)
   }
 
-  def fetch(
-    repositories: Seq[core.Repository]
-  )(implicit
-    cachePolicy: CachePolicy
-  ): ResolutionProcess.Fetch[Task] = {
-
-    modVers =>
-      Task.gatherUnordered(
-        modVers
-          .map {case (module, version) =>
-            Repository.find(repositories, module, version)
-              .run
-              .map((module, version) -> _)
-          }
-      )
-  }
-
-  implicit def fetchLocalFirst(
-    repositories: Seq[core.Repository]
-  )(implicit
-    cachePolicy: CachePolicy
-  ): ResolutionProcess.Fetch[Task] = {
-
-    modVers =>
-      Task.gatherUnordered(
-        modVers
-          .map {case (module, version) =>
-            def attempt(cachePolicy: CachePolicy) =
-              Repository.find(repositories, module, version)(cachePolicy)
-                .run
-                .map((module, version) -> _)
-
-            attempt(CachePolicy.LocalOnly).flatMap {
-              case v @ (_, \/-(_)) => Task.now(v)
-              case (_, -\/(_)) => attempt(cachePolicy)
-            }
-          }
-      )
-  }
-
 }

core/shared/src/main/scala/coursier/util/Xml.scala

@@ -0,0 +1,25 @@
+package coursier.util
+
+object Xml {
+
+  /** A representation of an XML node/document, with different implementations on JVM and JS */
+  trait Node {
+    def label: String
+    def child: Seq[Node]
+    def isText: Boolean
+    def textContent: String
+    def isElement: Boolean
+  }
+
+  object Node {
+    val empty: Node =
+      new Node {
+        val isText = false
+        val isElement = false
+        val child = Nil
+        val label = ""
+        val textContent = ""
+      }
+  }
+
+}

fetch-js/src/main/scala/coursier/Fetch.scala

@@ -0,0 +1,26 @@
+package coursier
+
+import scalaz.concurrent.Task
+
+object Fetch {
+
+  implicit def default(
+    repositories: Seq[core.Repository]
+  ): ResolutionProcess.Fetch[Task] =
+    apply(repositories, Platform.artifact)
+
+  def apply(
+    repositories: Seq[core.Repository],
+    fetch: Repository.Fetch[Task]
+  ): ResolutionProcess.Fetch[Task] = {
+
+    modVers => Task.gatherUnordered(
+      modVers.map { case (module, version) =>
+        Repository.find(repositories, module, version, fetch)
+          .run
+          .map((module, version) -> _)
+      }
+    )
+  }
+
+}

fetch-js/src/main/scala/coursier/Platform.scala

@@ -1,18 +1,17 @@
 package coursier
-package core
 
 import org.scalajs.dom.raw.{ Event, XMLHttpRequest }
 
 import scala.concurrent.{ ExecutionContext, Promise, Future }
-import scalaz.{ -\/, \/-, EitherT }
-import scalaz.concurrent.Task
 
 import scala.scalajs.js
 import js.Dynamic.{ global => g }
 
 import scala.scalajs.js.timers._
+import scalaz.concurrent.Task
+import scalaz.{ -\/, \/-, EitherT }
 
-object MavenRepository {
+object Platform {
 
   def encodeURIComponent(s: String): String =
     g.encodeURIComponent(s).asInstanceOf[String]
@@ -76,37 +75,34 @@ object MavenRepository {
       p.future
     }
 
+  val artifact: Repository.Fetch[Task] = { artifact =>
+    EitherT(
+      Task { implicit ec =>
+        get(artifact.url)
+          .map(\/-(_))
+          .recover { case e: Exception =>
+          -\/(e.getMessage)
+        }
+      }
+    )
+  }
+
   trait Logger {
     def fetching(url: String): Unit
     def fetched(url: String): Unit
     def other(url: String, msg: String): Unit
   }
 
-}
+  def artifactWithLogger(logger: Logger): Repository.Fetch[Task] = { artifact =>
-
-case class MavenRepository(
-  root: String,
-  ivyLike: Boolean = false,
-  logger: Option[MavenRepository.Logger] = None
-) extends BaseMavenRepository(root, ivyLike) {
-
-
-  def fetch(
-    artifact: Artifact,
-    cachePolicy: CachePolicy
-  ): EitherT[Task, String, String] = {
-
-    val url0 = root + artifact.url
-
     EitherT(
       Task { implicit ec =>
-        Future(logger.foreach(_.fetching(url0)))
+        Future(logger.fetching(artifact.url))
-          .flatMap(_ => MavenRepository.get(url0))
+          .flatMap(_ => get(artifact.url))
-          .map{ s => logger.foreach(_.fetched(url0)); \/-(s) }
+          .map { s => logger.fetched(artifact.url); \/-(s) }
-          .recover{case e: Exception =>
+          .recover { case e: Exception =>
-            logger.foreach(_.other(url0, e.getMessage))
+          logger.other(artifact.url, e.getMessage)
-            -\/(e.getMessage)
+          -\/(e.getMessage)
-          }
+        }
       }
     )
   }

files/src/main/scala/coursier/Cache.scala

@@ -1,9 +1,6 @@
 package coursier
 
-import java.io.{PrintWriter, File}
+import java.io.{ File, PrintWriter }
-
-import coursier.core.MavenRepository
-
 import scala.io.Source
 
 object Cache {
@@ -93,10 +90,9 @@ case class Cache(cache: File) {
       .flatMap { f =>
         val name = f.getName
         val lines = Source.fromFile(f).getLines().toList
-        mavenRepository(lines)
+        mavenRepository(lines).map(repo =>
-          .map(repo =>
+          (name, repo, (repo.root, new File(fileBase, name)))
-            (name, repo.copy(cache = Some(new File(metadataBase, name))), (repo.root, new File(fileBase, name)))
+        )
-          )
       }
 
   def map(): Map[String, (MavenRepository, (String, File))] =

files/src/main/scala/coursier/CachePolicy.scala

@@ -0,0 +1,49 @@
+package coursier
+
+import scalaz.\/
+import scalaz.concurrent.Task
+
+sealed trait CachePolicy {
+  def apply[T](
+    tryRemote: T => Boolean )(
+    local: => Task[T] )(
+    remote: Option[T] => Task[T]
+  ): Task[T]
+}
+
+object CachePolicy {
+  def saving[E,T](
+    remote: => Task[E \/ T] )(
+    save: T => Task[Unit]
+  ): Task[E \/ T] = {
+    for {
+      res <- remote
+      _ <- res.fold(_ => Task.now(()), t => save(t))
+    } yield res
+  }
+
+  case object Default extends CachePolicy {
+    def apply[T](
+      tryRemote: T => Boolean )(
+      local: => Task[T] )(
+      remote: Option[T] => Task[T]
+    ): Task[T] =
+      local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res))
+  }
+  case object LocalOnly extends CachePolicy {
+    def apply[T](
+      tryRemote: T => Boolean )(
+      local: => Task[T] )(
+      remote: Option[T] => Task[T]
+    ): Task[T] =
+      local
+  }
+  case object ForceDownload extends CachePolicy {
+    def apply[T](
+      tryRemote: T => Boolean )(
+      local: => Task[T] )(
+      remote: Option[T] => Task[T]
+    ): Task[T] =
+      remote(None)
+  }
+}

files/src/main/scala/coursier/Fetch.scala

@@ -0,0 +1,26 @@
+package coursier
+
+import scalaz.concurrent.Task
+
+object Fetch {
+
+  implicit def default(
+    repositories: Seq[core.Repository]
+  ): ResolutionProcess.Fetch[Task] =
+    apply(repositories, Platform.artifact)
+
+  def apply(
+    repositories: Seq[core.Repository],
+    fetch: Repository.Fetch[Task]
+  ): ResolutionProcess.Fetch[Task] = {
+
+    modVers => Task.gatherUnordered(
+      modVers.map { case (module, version) =>
+        Repository.find(repositories, module, version, fetch)
+          .run
+          .map((module, version) -> _)
+      }
+    )
+  }
+
+}

files/src/main/scala/coursier/Files.scala

@@ -22,18 +22,18 @@ case class Files(
     Executors.newFixedThreadPool(concurrentDownloadCount, Strategy.DefaultDaemonThreadFactory)
 
   def withLocal(artifact: Artifact): Artifact = {
-    val isLocal =
-      artifact.url.startsWith("file:/") &&
-        artifact.checksumUrls.values.forall(_.startsWith("file:/"))
-
     def local(url: String) =
       if (url.startsWith("file:///"))
         url.stripPrefix("file://")
       else if (url.startsWith("file:/"))
         url.stripPrefix("file:")
       else
-        cache.find{case (base, _) => url.startsWith(base)} match {
+        cache.find { case (base, _) => url.startsWith(base) } match {
-          case None => ???
+          case None =>
+            // FIXME Means we were handed an artifact from repositories other than the known ones
+            println(cache.mkString("\n"))
+            println(url)
+            ???
           case Some((base, cacheDir)) =>
             cacheDir + "/" + url.stripPrefix(base)
         }
@@ -95,7 +95,13 @@ case class Files(
           logger.foreach(_.downloadingArtifact(url))
 
           val url0 = new URL(url)
-          val in = new BufferedInputStream(url0.openStream(), Files.bufferSize)
+          val conn = url0.openConnection() // FIXME Should this be closed?
+          // Dummy user-agent instead of the default "Java/...",
+          // so that we are not returned incomplete/erroneous metadata
+          // (Maven 2 compatibility? - happens for snapshot versioning metadata,
+          // this is SO FUCKING CRAZY)
+          conn.setRequestProperty("User-Agent", "")
+          val in = new BufferedInputStream(conn.getInputStream(), Files.bufferSize)
 
           val result =
             try {
@@ -141,19 +147,26 @@ case class Files(
 
 
     val tasks =
-      for ((f, url) <- pairs) yield
+      for ((f, url) <- pairs) yield {
+        val file = new File(f)
+
         if (url != ("file:" + f) && url != ("file://" + f)) {
           assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url")
-          val file = new File(f)
           cachePolicy[FileError \/ File](
             _.isLeft)(
             locally(file))(
             _ => remote(file, url)
           ).map(e => (file, url) -> e.map(_ => ()))
-        } else {
+        } else
-          val file = new File(f)
+          Task {
-          Task.now(((file, url), \/-(())))
+            (file, url) -> {
-        }
+              if (file.exists())
+                \/-(())
+              else
+                -\/(FileError.NotFound(file.toString))
+            }
+          }
+      }
 
     Nondeterminism[Task].gather(tasks)
   }
@@ -205,28 +218,42 @@ case class Files(
     cachePolicy: CachePolicy,
     pool: ExecutorService = defaultPool
   ): EitherT[Task, FileError, File] =
-    EitherT{
+    EitherT {
-      val res =
+      val res = download(artifact).map { results =>
-        download(artifact)
+        val ((f, _), res) = results.head
-          .map(results =>
+        res.map(_ => f)
-            results.head._2.map(_ => results.head._1._1)
+      }
-          )
 
       checksum.fold(res) { sumType =>
-        res
+        res.flatMap {
-          .flatMap{
+          case err @ -\/(_) => Task.now(err)
-            case err @ -\/(_) => Task.now(err)
+          case \/-(f) =>
-            case \/-(f) =>
+            validateChecksum(artifact, sumType)
-              validateChecksum(artifact, sumType)
+              .map(_.map(_ => f))
-                .map(_.map(_ => f))
+        }
-          }
       }
     }
 
+  def fetch(implicit
+    cachePolicy: CachePolicy,
+    pool: ExecutorService = defaultPool
+  ): Repository.Fetch[Task] = {
+    artifact =>
+      file(artifact)(cachePolicy).leftMap(_.message).map { f =>
+        // FIXME Catch error here?
+        scala.io.Source.fromFile(f)("UTF-8").mkString
+      }
+  }
+
 }
 
 object Files {
-  
+
+  lazy val ivy2Local = MavenRepository(
+    new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString,
+    ivyLike = true
+  )
+
   val defaultConcurrentDownloadCount = 6
 
   // FIXME This kind of side-effecting API is lame, we should aim at a more functional one.
@@ -276,14 +303,26 @@ object Files {
 
 }
 
-sealed trait FileError
+sealed trait FileError {
+  def message: String
+}
 
 object FileError {
 
-  case class DownloadError(message: String) extends FileError
+  case class DownloadError(message0: String) extends FileError {
-  case class NotFound(file: String) extends FileError
+    def message = s"Download error: $message0"
-  case class Locked(file: String) extends FileError
+  }
-  case class ChecksumNotFound(sumType: String, file: String) extends FileError
+  case class NotFound(file: String) extends FileError {
-  case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError
+    def message = s"$file: not found"
+  }
+  case class Locked(file: String) extends FileError {
+    def message = s"$file: locked"
+  }
+  case class ChecksumNotFound(sumType: String, file: String) extends FileError {
+    def message = s"$file: $sumType checksum not found"
+  }
+  case class WrongChecksum(sumType: String, got: String, expected: String, file: String, sumFile: String) extends FileError {
+    def message = s"$file: $sumType checksum validation failed"
+  }
 
 }

files/src/main/scala/coursier/Platform.scala

@@ -0,0 +1,56 @@
+package coursier
+
+import java.io._
+import java.net.URL
+
+import scalaz._
+import scalaz.concurrent.Task
+
+object Platform {
+
+  def readFullySync(is: InputStream) = {
+    val buffer = new ByteArrayOutputStream()
+    val data = Array.ofDim[Byte](16384)
+
+    var nRead = is.read(data, 0, data.length)
+    while (nRead != -1) {
+      buffer.write(data, 0, nRead)
+      nRead = is.read(data, 0, data.length)
+    }
+
+    buffer.flush()
+    buffer.toByteArray
+  }
+
+  def readFully(is: => InputStream) =
+    Task {
+      \/.fromTryCatchNonFatal {
+        val is0 = is
+        val b =
+          try readFullySync(is0)
+          finally is0.close()
+
+        new String(b, "UTF-8")
+      } .leftMap{
+        case e: java.io.FileNotFoundException =>
+          s"Not found: ${e.getMessage}"
+        case e =>
+          s"$e: ${e.getMessage}"
+      }
+    }
+
+  val artifact: Repository.Fetch[Task] = { artifact =>
+    EitherT {
+      val url = new URL(artifact.url)
+
+      val conn = url.openConnection()
+      // Dummy user-agent instead of the default "Java/...",
+      // so that we are not returned incomplete/erroneous metadata
+      // (Maven 2 compatibility? - happens for snapshot versioning metadata,
+      // this is SO FUCKING CRAZY)
+      conn.setRequestProperty("User-Agent", "")
+      readFully(conn.getInputStream())
+    }
+  }
+
+}

tests/js/src/test/scala/coursier/test/JsTests.scala

@@ -1,12 +1,11 @@
 package coursier
 package test
 
-import coursier.core.{Repository, MavenRepository}
 import coursier.test.compatibility._
 
 import utest._
 
-import scala.concurrent.{Future, Promise}
+import scala.concurrent.{ Future, Promise }
 
 object JsTests extends TestSuite {
 
@@ -18,7 +17,7 @@ object JsTests extends TestSuite {
     }
 
     'get{
-      MavenRepository.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom")
+      Platform.get("http://repo1.maven.org/maven2/ch/qos/logback/logback-classic/1.1.3/logback-classic-1.1.3.pom")
         .map(core.compatibility.xmlParse)
         .map{ xml =>
           assert(xml.right.toOption.exists(_.label == "project"))
@@ -26,10 +25,8 @@ object JsTests extends TestSuite {
     }
 
     'getProj{
-      implicit val cachePolicy = CachePolicy.Default
+      MavenRepository("https://repo1.maven.org/maven2/")
-
+        .find(Module("ch.qos.logback", "logback-classic"), "1.1.3", Platform.artifact)
-      Repository.mavenCentral
-        .find(Module("ch.qos.logback", "logback-classic"), "1.1.3")
         .map{case (_, proj) =>
           assert(proj.parent == Some(Module("ch.qos.logback", "logback-parent"), "1.1.3"))
         }

tests/js/src/test/scala/coursier/test/compatibility/package.scala

@@ -13,7 +13,7 @@ package object compatibility {
   def textResource(path: String)(implicit ec: ExecutionContext): Future[String] = {
     val p = Promise[String]()
 
-    fs.readFile("core/shared/src/test/resources/" + path, "utf-8", {
+    fs.readFile("tests/shared/src/test/resources/" + path, "utf-8", {
       (err: js.Dynamic, data: js.Dynamic) =>
         if (err == null) p.success(data.asInstanceOf[String])
         else p.failure(new Exception(err.toString))

tests/jvm/src/test/scala/coursier/test/IvyLocalTests.scala

@@ -1,7 +1,6 @@
 package coursier.test
 
-import coursier.Module
+import coursier.{ Module, Files }
-import coursier.core.Repository
 import utest._
 
 object IvyLocalTests extends TestSuite {
@@ -11,7 +10,7 @@ object IvyLocalTests extends TestSuite {
       // Assume this module (and the sub-projects it depends on) is published locally
       CentralTests.resolutionCheck(
         Module("com.github.alexarchambault", "coursier_2.11"), "0.1.0-SNAPSHOT",
-        Some(Repository.ivy2Local))
+        Some(Files.ivy2Local))
     }
   }
 

tests/jvm/src/test/scala/coursier/test/compatibility/package.scala

@@ -1,6 +1,6 @@
 package coursier.test
 
-import coursier.core.MavenRepository
+import coursier.Platform
 
 import scala.concurrent.{ExecutionContext, Future}
 import scalaz.concurrent.Task
@@ -19,7 +19,7 @@ package object compatibility {
       .getResource(path)
       .openStream()
 
-    new String(MavenRepository.readFullySync(is), "UTF-8")
+    new String(Platform.readFullySync(is), "UTF-8")
   }
 
 }

tests/shared/src/test/resources/resolutions/com.github.alexarchambault/coursier_2.11/0.1.0-SNAPSHOT

@@ -2,6 +2,4 @@ com.github.alexarchambault:coursier_2.11:jar:0.1.0-SNAPSHOT
 org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4
 org.scala-lang.modules:scala-xml_2.11:jar:1.0.4
 org.scala-lang:scala-library:jar:2.11.7
-org.scalaz:scalaz-concurrent_2.11:jar:7.1.2
 org.scalaz:scalaz-core_2.11:jar:7.1.2
-org.scalaz:scalaz-effect_2.11:jar:7.1.2

tests/shared/src/test/scala/coursier/test/CentralTests.scala

@@ -1,20 +1,18 @@
 package coursier
 package test
 
-import coursier.core.{ Repository, MavenRepository }
 import utest._
 import scala.async.Async.{ async, await }
 
+import coursier.Fetch.default
 import coursier.test.compatibility._
 
 object CentralTests extends TestSuite {
 
   val repositories = Seq[Repository](
-    Repository.mavenCentral
+    MavenRepository("https://repo1.maven.org/maven2/")
   )
 
-  implicit val cachePolicy = CachePolicy.Default
-
   def resolve(
     deps: Set[Dependency],
     filter: Option[Dependency => Boolean] = None,

tests/shared/src/test/scala/coursier/test/PomParsingTests.scala

@@ -4,7 +4,8 @@ package test
 import utest._
 import scalaz._
 
-import coursier.core.Xml
+import coursier.maven.Pom
+
 import coursier.core.compatibility._
 
 object PomParsingTests extends TestSuite {
@@ -22,7 +23,7 @@ object PomParsingTests extends TestSuite {
 
       val expected = \/-(Dependency(Module("comp", "lib"), "2.1", attributes = Attributes(classifier = "extra")))
 
-      val result = Xml.dependency(xmlParse(depNode).right.get)
+      val result = Pom.dependency(xmlParse(depNode).right.get)
 
       assert(result == expected)
     }
@@ -35,7 +36,7 @@ object PomParsingTests extends TestSuite {
 
       val expected = \/-(Profile("profile1", None, Profile.Activation(Nil), Nil, Nil, Map.empty))
 
-      val result = Xml.profile(xmlParse(profileNode).right.get)
+      val result = Pom.profile(xmlParse(profileNode).right.get)
 
       assert(result == expected)
     }
@@ -50,7 +51,7 @@ object PomParsingTests extends TestSuite {
 
       val expected = \/-(Profile("", Some(true), Profile.Activation(Nil), Nil, Nil, Map.empty))
 
-      val result = Xml.profile(xmlParse(profileNode).right.get)
+      val result = Pom.profile(xmlParse(profileNode).right.get)
 
       assert(result == expected)
     }
@@ -66,7 +67,7 @@ object PomParsingTests extends TestSuite {
 
       val expected = \/-(Profile("profile1", Some(true), Profile.Activation(Nil), Nil, Nil, Map.empty))
 
-      val result = Xml.profile(xmlParse(profileNode).right.get)
+      val result = Pom.profile(xmlParse(profileNode).right.get)
 
       assert(result == expected)
     }
@@ -94,7 +95,7 @@ object PomParsingTests extends TestSuite {
         Map.empty
       ))
 
-      val result = Xml.profile(xmlParse(profileNode).right.get)
+      val result = Pom.profile(xmlParse(profileNode).right.get)
 
       assert(result == expected)
     }
@@ -125,7 +126,7 @@ object PomParsingTests extends TestSuite {
         Map.empty
       ))
 
-      val result = Xml.profile(xmlParse(profileNode).right.get)
+      val result = Pom.profile(xmlParse(profileNode).right.get)
 
       assert(result == expected)
     }
@@ -148,7 +149,7 @@ object PomParsingTests extends TestSuite {
         Map("first.prop" -> "value1")
       ))
 
-      val result = Xml.profile(xmlParse(profileNode).right.get)
+      val result = Pom.profile(xmlParse(profileNode).right.get)
 
       assert(result == expected)
     }
@@ -194,7 +195,7 @@ object PomParsingTests extends TestSuite {
       assert(node.label == "properties")
 
       val children = node.child.collect{case elem if elem.isElement => elem}
-      val props0 = children.toList.traverseU(Xml.property)
+      val props0 = children.toList.traverseU(Pom.property)
 
       assert(props0.isRight)
 

tests/shared/src/test/scala/coursier/test/ResolutionTests.scala

@@ -3,18 +3,16 @@ package test
 
 import coursier.core.Repository
 import utest._
-import scala.async.Async.{async, await}
+import scala.async.Async.{ async, await }
 
 import coursier.test.compatibility._
 
 object ResolutionTests extends TestSuite {
 
-  implicit val cachePolicy = CachePolicy.Default
-
   def resolve0(deps: Set[Dependency], filter: Option[Dependency => Boolean] = None) = {
     Resolution(deps, filter = filter)
       .process
-      .run(fetch(repositories))
+      .run(Fetch.default(repositories))
       .runF
   }
 

tests/shared/src/test/scala/coursier/test/TestRepository.scala

@@ -3,16 +3,21 @@ package test
 
 import coursier.core._
 
-import scalaz.EitherT
+import scalaz.{ Monad, EitherT }
-import scalaz.concurrent.Task
 import scalaz.Scalaz._
 
 class TestRepository(projects: Map[(Module, String), Project]) extends Repository {
   val source = new core.Artifact.Source {
     def artifacts(dependency: Dependency, project: Project) = ???
   }
-  def find(module: Module, version: String)(implicit cachePolicy: CachePolicy) =
+  def find[F[_]](
-    EitherT(Task.now(
+    module: Module,
+    version: String,
+    fetch: Repository.Fetch[F]
+  )(implicit
+    F: Monad[F]
+  ) =
+    EitherT(F.point(
       projects.get((module, version)).map((source, _)).toRightDisjunction("Not found")
     ))
 }

web/src/main/scala/coursier/web/Backend.scala

@@ -1,7 +1,8 @@
 package coursier
 package web
 
-import coursier.core.{ Repository, MavenRepository, MavenSource }
+import coursier.maven.MavenSource
+
 import japgolly.scalajs.react.vdom.{ TagMod, Attr }
 import japgolly.scalajs.react.vdom.Attrs.dangerouslySetInnerHtml
 import japgolly.scalajs.react.{ ReactEventI, ReactComponentB, BackendScope }
@@ -10,6 +11,7 @@ import scala.scalajs.concurrent.JSExecutionContext.Implicits.queue
 import org.scalajs.jquery.jQuery
 
 import scala.concurrent.Future
+import scalaz.concurrent.Task
 
 import scala.scalajs.js
 import js.Dynamic.{ global => g }
@@ -32,6 +34,22 @@ case class State(
 )
 
 class Backend($: BackendScope[Unit, State]) {
+
+  def fetch(
+    repositories: Seq[core.Repository],
+    fetch: Repository.Fetch[Task]
+  ): ResolutionProcess.Fetch[Task] = {
+
+    modVers => Task.gatherUnordered(
+      modVers.map { case (module, version) =>
+        Repository.find(repositories, module, version, fetch)
+          .run
+          .map((module, version) -> _)
+      }
+    )
+  }
+
+
   def updateDepGraph(resolution: Resolution) = {
     println("Rendering canvas")
 
@@ -138,7 +156,7 @@ class Backend($: BackendScope[Unit, State]) {
     g.$("#resLogTab a:last").tab("show")
     $.modState(_.copy(resolving = true, log = Nil))
 
-    val logger: MavenRepository.Logger = new MavenRepository.Logger {
+    val logger: Platform.Logger = new Platform.Logger {
       def fetched(url: String) = {
         println(s"<- $url")
         $.modState(s => s.copy(log = s"<- $url" +: s.log))
@@ -163,11 +181,9 @@ class Backend($: BackendScope[Unit, State]) {
         )
       )
 
-      implicit val cachePolicy = CachePolicy.Default
-
       res
         .process
-        .run(s.repositories.map(item => item._2.copy(logger = Some(logger))), 100)
+        .run(fetch(s.repositories.map { case (_, repo) => repo }, Platform.artifactWithLogger(logger)), 100)
     }
 
     // For reasons that are unclear to me, not delaying this when using the runNow execution context
@@ -702,7 +718,7 @@ object App {
 
   val initialState = State(
     Nil,
-    Seq("central" -> Repository.mavenCentral),
+    Seq("central" -> MavenRepository("https://repo1.maven.org/maven2/")),
     ResolutionOptions(),
     None,
     -1,