Merge pull request #189 from alexarchambault/topic/url-protocol

Add support for custom URL protocols, change the way cache directory is specified

build.sbt

@@ -220,6 +220,13 @@ lazy val cache = project
       import com.typesafe.tools.mima.core.ProblemFilters._
       
       Seq(
+        // Since 1.0.0-M10
+        // cache argument type changed from `Seq[(String, File)]` to `File`
+        ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.file"),
+        ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.fetch"),
+        ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.Cache.default"),
+        ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.validateChecksum"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.defaultBase"),
         // Since 1.0.0-M9
         // Added an optional extra parameter to FileError.NotFound - only
         // its unapply method should break compatibility at the source level.

cache/src/main/scala/coursier/Cache.scala

@@ -1,7 +1,7 @@
 package coursier
 
 import java.math.BigInteger
-import java.net.{ HttpURLConnection, URL, URLConnection }
+import java.net.{ HttpURLConnection, URL, URLConnection, URLStreamHandler }
 import java.nio.channels.{ OverlappingFileLockException, FileLock }
 import java.nio.file.{ StandardCopyOption, Files => NioFiles }
 import java.security.MessageDigest
@@ -43,25 +43,30 @@ object Cache {
     }
   }
 
-  private def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = {
+  private def withLocal(artifact: Artifact, cache: File): Artifact = {
     def local(url: String) =
       if (url.startsWith("file:///"))
         url.stripPrefix("file://")
       else if (url.startsWith("file:/"))
         url.stripPrefix("file:")
-      else {
+      else
-        val localPathOpt = cache.collectFirst {
+        // FIXME Should we fully parse the URL here?
-          case (base, cacheDir) if url.startsWith(base) =>
+        // FIXME Should some safeguards be added against '..' components in paths?
-            cacheDir.toString + "/" + escape(url.stripPrefix(base))
+        url.split(":", 2) match {
-        }
+          case Array(protocol, remaining) =>
+            val remaining0 =
+              if (remaining.startsWith("///"))
+                remaining.stripPrefix("///")
+              else if (remaining.startsWith("/"))
+                remaining.stripPrefix("/")
+              else
+                throw new Exception(s"URL $url doesn't contain an absolute path")
 
-        localPathOpt.getOrElse {
+            new File(cache, escape(protocol + "/" + remaining0)) .toString
-          // FIXME Means we were handed an artifact from repositories other than the known ones
+
-          println(cache.mkString("\n"))
+          case _ =>
-          println(url)
+            throw new Exception(s"No protocol found in URL $url")
-          ???
         }
-      }
 
     if (artifact.extra.contains("local"))
       artifact
@@ -178,9 +183,67 @@ object Cache {
 
   private val partialContentResponseCode = 206
 
+  private val handlerClsCache = new ConcurrentHashMap[String, Option[URLStreamHandler]]
+
+  private def handlerFor(url: String): Option[URLStreamHandler] = {
+    val protocol = url.takeWhile(_ != ':')
+
+    Option(handlerClsCache.get(protocol)) match {
+      case None =>
+        val clsName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
+        val clsOpt =
+          try Some(Thread.currentThread().getContextClassLoader.loadClass(clsName))
+          catch {
+            case _: ClassNotFoundException =>
+              None
+          }
+
+        def printError(e: Exception): Unit =
+          scala.Console.err.println(
+            s"Cannot instantiate $clsName: $e${Option(e.getMessage).map(" ("+_+")")}"
+          )
+
+        val handlerOpt = clsOpt.flatMap {
+          cls =>
+            try Some(cls.newInstance().asInstanceOf[URLStreamHandler])
+            catch {
+              case e: InstantiationException =>
+                printError(e)
+                None
+              case e: IllegalAccessException =>
+                printError(e)
+                None
+              case e: ClassCastException =>
+                printError(e)
+                None
+            }
+        }
+
+        val prevOpt = Option(handlerClsCache.putIfAbsent(protocol, handlerOpt))
+        prevOpt.getOrElse(handlerOpt)
+
+      case Some(handlerOpt) =>
+        handlerOpt
+    }
+  }
+
+  /**
+    * Returns a `java.net.URL` for `s`, possibly using the custom protocol handlers found under the
+    * `coursier.cache.protocol` namespace.
+    *
+    * E.g. URL `"test://abc.com/foo"`, having protocol `"test"`, can be handled by a
+    * `URLStreamHandler` named `coursier.cache.protocol.TestHandler` (protocol name gets
+    * capitalized, and suffixed with `Handler` to get the class name).
+    *
+    * @param s
+    * @return
+    */
+  def url(s: String): URL =
+    new URL(null, s, handlerFor(s).orNull)
+
   private def download(
     artifact: Artifact,
-    cache: Seq[(String, File)],
+    cache: File,
     checksums: Set[String],
     cachePolicy: CachePolicy,
     pool: ExecutorService,
@@ -213,8 +276,8 @@ object Cache {
           .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
       }
 
-    def urlConn(url: String) = {
+    def urlConn(url0: String) = {
-      val conn = new URL(url).openConnection() // FIXME Should this be closed?
+      val conn = url(url0).openConnection() // FIXME Should this be closed?
       // Dummy user-agent instead of the default "Java/...",
       // so that we are not returned incomplete/erroneous metadata
       // (Maven 2 compatibility? - happens for snapshot versioning metadata,
@@ -460,7 +523,7 @@ object Cache {
   def validateChecksum(
     artifact: Artifact,
     sumType: String,
-    cache: Seq[(String, File)],
+    cache: File,
     pool: ExecutorService
   ): EitherT[Task, FileError, Unit] = {
 
@@ -514,7 +577,7 @@ object Cache {
 
   def file(
     artifact: Artifact,
-    cache: Seq[(String, File)] = default,
+    cache: File = default,
     cachePolicy: CachePolicy = CachePolicy.FetchMissing,
     checksums: Seq[Option[String]] = defaultChecksums,
     logger: Option[Logger] = None,
@@ -565,7 +628,7 @@ object Cache {
   }
 
   def fetch(
-    cache: Seq[(String, File)] = default,
+    cache: File = default,
     cachePolicy: CachePolicy = CachePolicy.FetchMissing,
     checksums: Seq[Option[String]] = defaultChecksums,
     logger: Option[Logger] = None,
@@ -613,18 +676,13 @@ object Cache {
     dropInfoAttributes = true
   )
 
-  lazy val defaultBase = new File(
+  lazy val default = new File(
     sys.env.getOrElse(
       "COURSIER_CACHE",
       sys.props("user.home") + "/.coursier/cache/v1"
     )
   ).getAbsoluteFile
 
-  lazy val default = Seq(
-    "http://" -> new File(defaultBase, "http"),
-    "https://" -> new File(defaultBase, "https")
-  )
-
   val defaultConcurrentDownloadCount = 6
 
   lazy val defaultPool =

cache/src/main/scala/coursier/CacheParse.scala

@@ -1,5 +1,7 @@
 package coursier
 
+import java.net.MalformedURLException
+
 import coursier.ivy.IvyRepository
 import coursier.util.Parse
 
@@ -24,10 +26,13 @@ object CacheParse {
           sys.error(s"Unrecognized repository: $r")
       }
 
-      if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/"))
+      try {
+        Cache.url(url)
         repo.success
-      else
+      } catch {
-        s"Unrecognized protocol in $url".failure
+        case e: MalformedURLException =>
+          ("Error parsing URL " + url + Option(e.getMessage).map(" (" + _ + ")").mkString).failure
+      }
     }
 
   def repositories(l: Seq[String]): ValidationNel[String, Seq[Repository]] =

cache/src/main/scala/coursier/Platform.scala

@@ -41,7 +41,7 @@ object Platform {
 
   val artifact: Fetch.Content[Task] = { artifact =>
     EitherT {
-      val url = new URL(artifact.url)
+      val url = Cache.url(artifact.url)
 
       val conn = url.openConnection()
       // Dummy user-agent instead of the default "Java/...",

cli/src/main/scala-2.11/coursier/cli/Coursier.scala

@@ -74,7 +74,7 @@ case class CommonOptions(
 case class CacheOptions(
   @Help("Cache directory (defaults to environment variable COURSIER_CACHE or ~/.coursier/cache/v1)")
   @Short("C")
-    cache: String = Cache.defaultBase.toString
+    cache: String = Cache.default.toString
 )
 
 sealed abstract class CoursierCommand extends Command
@@ -490,9 +490,9 @@ case class Bootstrap(
   val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v }
   val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v }
 
-  val unrecognized = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://"))
+  val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://"))
-  if (unrecognized.nonEmpty)
+  if (nonHttpUrls.nonEmpty)
-    Console.err.println(s"Warning: non HTTP URLs:\n${unrecognized.mkString("\n")}")
+    Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}")
 
   val buffer = new ByteArrayOutputStream()
 

cli/src/main/scala-2.11/coursier/cli/Helper.scala

@@ -77,11 +77,7 @@ class Helper(
       )
   }
 
-  val caches =
+  val cache = new File(cacheOptions.cache)
-    Seq(
-      "http://" -> new File(new File(cacheOptions.cache), "http"),
-      "https://" -> new File(new File(cacheOptions.cache), "https")
-    )
 
   val pool = Executors.newFixedThreadPool(parallel, Strategy.DefaultDaemonThreadFactory)
 
@@ -200,7 +196,7 @@ class Helper(
       None
 
   val fetchs = cachePolicies.map(p =>
-    Cache.fetch(caches, p, checksums = checksums, logger = logger, pool = pool)
+    Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool)
   )
   val fetchQuiet = coursier.Fetch.from(
     repositories,
@@ -332,8 +328,8 @@ class Helper(
       println(s"  Found ${artifacts0.length} artifacts")
 
     val tasks = artifacts0.map(artifact =>
-      (Cache.file(artifact, caches, cachePolicies.head, checksums = checksums, logger = logger, pool = pool) /: cachePolicies.tail)(
+      (Cache.file(artifact, cache, cachePolicies.head, checksums = checksums, logger = logger, pool = pool) /: cachePolicies.tail)(
-        _ orElse Cache.file(artifact, caches, _, checksums = checksums, logger = logger, pool = pool)
+        _ orElse Cache.file(artifact, cache, _, checksums = checksums, logger = logger, pool = pool)
       ).run.map(artifact.->)
     )
 

plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala

@@ -39,7 +39,7 @@ object CoursierPlugin extends AutoPlugin {
     coursierVerbosity := 1,
     coursierResolvers <<= Tasks.coursierResolversTask,
     coursierSbtResolvers <<= externalResolvers in updateSbtClassifiers,
-    coursierCache := Cache.defaultBase,
+    coursierCache := Cache.default,
     update <<= Tasks.updateTask(withClassifiers = false),
     updateClassifiers <<= Tasks.updateTask(withClassifiers = true),
     updateSbtClassifiers in Defaults.TaskGlobal <<= Tasks.updateTask(withClassifiers = true, sbtClassifiers = true),

plugin/src/main/scala-2.10/coursier/FromSbt.scala

@@ -1,8 +1,11 @@
 package coursier
 
 import coursier.ivy.{ IvyXml, IvyRepository }
-import sbt.mavenint.SbtPomExtraProperties
+
+import java.net.MalformedURLException
+
 import sbt.{ Resolver, CrossVersion, ModuleID }
+import sbt.mavenint.SbtPomExtraProperties
 
 object FromSbt {
 
@@ -105,12 +108,20 @@ object FromSbt {
   def repository(resolver: Resolver, ivyProperties: Map[String, String]): Option[Repository] =
     resolver match {
       case sbt.MavenRepository(_, root) =>
-        if (root.startsWith("http://") || root.startsWith("https://") || root.startsWith("file:/")) {
+        try {
+          Cache.url(root) // ensure root is a URL whose protocol can be handled here
           val root0 = if (root.endsWith("/")) root else root + "/"
           Some(MavenRepository(root0, sbtAttrStub = true))
-        } else {
+        } catch {
-          Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it")
+          case e: MalformedURLException =>
-          None
+            Console.err.println(
+              "Warning: error parsing Maven repository base " +
+              root +
+              Option(e.getMessage).map(" ("+_+")").mkString +
+              ", ignoring it"
+            )
+
+            None
         }
 
       case sbt.FileRepository(_, _, patterns)

plugin/src/main/scala-2.10/coursier/Tasks.scala

@@ -215,7 +215,7 @@ object Tasks {
       val artifactsChecksums = coursierArtifactsChecksums.value
       val maxIterations = coursierMaxIterations.value
       val cachePolicy = coursierCachePolicy.value
-      val cacheDir = coursierCache.value
+      val cache = coursierCache.value
 
       val sv = scalaVersion.value // is this always defined? (e.g. for Java only projects?)
 
@@ -273,11 +273,6 @@ object Tasks {
 
         val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties))
 
-        val caches = Seq(
-          "http://" -> new File(cacheDir, "http"),
-          "https://" -> new File(cacheDir, "https")
-        )
-
         val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory)
 
         def createLogger() = new TermDisplay(new OutputStreamWriter(System.err))
@@ -286,8 +281,8 @@ object Tasks {
 
         val fetch = Fetch.from(
           repositories,
-          Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool),
+          Cache.fetch(cache, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool),
-          Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool)
+          Cache.fetch(cache, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool)
         )
 
         def depsRepr(deps: Seq[(String, Dependency)]) =
@@ -411,7 +406,7 @@ object Tasks {
         val artifactsLogger = createLogger()
 
         val artifactFileOrErrorTasks = allArtifacts.toVector.map { a =>
-          Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _))
+          Cache.file(a, cache, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _))
         }
 
         if (verbosity >= 0)

tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom

@@ -0,0 +1,83 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.github.alexarchambault</groupId>
+    <artifactId>coursier_2.11</artifactId>
+    <packaging>jar</packaging>
+    <description>coursier</description>
+    <url>https://github.com/alexarchambault/coursier</url>
+    <version>1.0.0-M9-test</version>
+    <licenses>
+        <license>
+            <name>Apache 2.0</name>
+            <url>http://opensource.org/licenses/Apache-2.0</url>
+            <distribution>repo</distribution>
+        </license>
+    </licenses>
+    <name>coursier</name>
+    <organization>
+        <name>com.github.alexarchambault</name>
+        <url>https://github.com/alexarchambault/coursier</url>
+    </organization>
+    <scm>
+        <connection>scm:git:github.com/alexarchambault/coursier.git</connection>
+        <developerConnection>scm:git:git@github.com:alexarchambault/coursier.git</developerConnection>
+        <url>github.com/alexarchambault/coursier.git</url>
+    </scm>
+    <developers>
+        <developer>
+            <id>alexarchambault</id>
+            <name>Alexandre Archambault</name>
+            <url>https://github.com/alexarchambault</url>
+        </developer>
+    </developers>
+    <dependencies>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>2.11.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.scoverage</groupId>
+            <artifactId>scalac-scoverage-runtime_2.11</artifactId>
+            <version>1.1.0</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scoverage</groupId>
+            <artifactId>scalac-scoverage-plugin_2.11</artifactId>
+            <version>1.1.0</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scalaz</groupId>
+            <artifactId>scalaz-core_2.11</artifactId>
+            <version>7.1.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.scala-lang.modules</groupId>
+            <artifactId>scala-xml_2.11</artifactId>
+            <version>1.0.3</version>
+        </dependency>
+    </dependencies>
+    <repositories>
+        <repository>
+            <id>sonatypereleases</id>
+            <name>sonatype-releases</name>
+            <url>https://oss.sonatype.org/content/repositories/releases/</url>
+            <layout>default</layout>
+        </repository>
+        <repository>
+            <id>ScalazBintrayRepo</id>
+            <name>Scalaz Bintray Repo</name>
+            <url>http://dl.bintray.com/scalaz/releases/</url>
+            <layout>default</layout>
+        </repository>
+        <repository>
+            <id>sonatypereleases</id>
+            <name>sonatype-releases</name>
+            <url>https://oss.sonatype.org/content/repositories/releases/</url>
+            <layout>default</layout>
+        </repository>
+    </repositories>
+</project>

tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.md5

@@ -0,0 +1 @@
+abd72a03c065a31bbe5ede5b16da98a9

tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.sha1

@@ -0,0 +1 @@
+4630461322d079ad7c53c4f2004ed9509ca046c0

tests/jvm/src/test/scala/coursier/cache/protocol/TestprotocolHandler.scala

@@ -0,0 +1,27 @@
+package coursier.cache.protocol
+
+import java.net.{ URL, URLConnection, URLStreamHandler }
+
+class TestprotocolHandler extends URLStreamHandler {
+  protected def openConnection(url: URL): URLConnection = {
+    val resPath = "/test-repo/http/abc.com" + url.getPath
+    val resURLOpt = Option(getClass.getResource(resPath))
+
+    resURLOpt match {
+      case None =>
+        new URLConnection(url) {
+          def connect() = throw new NoSuchElementException(s"Resource $resPath")
+        }
+      case Some(resURL) =>
+        resURL.openConnection()
+    }
+  }
+}
+
+object TestprotocolHandler {
+  val protocol = "testprotocol"
+
+  // get this namespace via a macro?
+  val expectedClassName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
+  assert(classOf[TestprotocolHandler].getName == expectedClassName)
+}

tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala

@@ -35,12 +35,9 @@ object ChecksumTests extends TestSuite {
 
     'artifact - {
 
-      val cachePath = getClass.getResource("/checksums").getPath
+      val cachePath = getClass.getResource("/test-repo").getPath
 
-      val cache = Seq(
+      val cache = new File(cachePath)
-        "http://" -> new File(cachePath),
-        "https://" -> new File(cachePath)
-      )
 
       def validate(artifact: Artifact, sumType: String) =
         Cache.validateChecksum(

tests/jvm/src/test/scala/coursier/test/CustomProtocolTests.scala

@@ -0,0 +1,85 @@
+package coursier
+package test
+
+import java.io.File
+import java.nio.file.Files
+
+import coursier.cache.protocol.TestprotocolHandler
+import utest._
+
+import scala.util.Try
+
+object CustomProtocolTests extends TestSuite {
+
+  val tests = TestSuite {
+
+    def check(extraMavenRepo: String): Unit = {
+
+      val tmpDir = Files.createTempDirectory("coursier-protocol-tests").toFile
+
+      def cleanTmpDir() = {
+        def delete(f: File): Boolean =
+          if (f.isDirectory) {
+            val removedContent = f.listFiles().map(delete).forall(x => x)
+            val removedDir = f.delete()
+
+            removedContent && removedDir
+          } else
+            f.delete()
+
+        if (!delete(tmpDir))
+          Console.err.println(s"Warning: unable to remove temporary directory $tmpDir")
+      }
+      
+      val res = try {
+        val fetch = Fetch.from(
+          Seq(
+            MavenRepository(extraMavenRepo),
+            MavenRepository("https://repo1.maven.org/maven2")
+          ),
+          Cache.fetch(
+            tmpDir
+          )
+        )
+
+        val startRes = Resolution(
+          Set(
+            Dependency(
+              Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test"
+            )
+          )
+        )
+
+        startRes.process.run(fetch).run
+      } finally {
+        cleanTmpDir()
+      }
+
+      val errors = res.errors
+
+      assert(errors.isEmpty)
+    }
+
+    // using scala-test would allow to put the below comments in the test names...
+
+    * - {
+      // test that everything's fine with standard protocols
+      val repoPath = new File(getClass.getResource("/test-repo/http/abc.com").getPath)
+      check(repoPath.toURI.toString)
+    }
+
+    * - {
+      // test the Cache.url method
+      val shouldFail = Try(Cache.url("notfoundzzzz://foo/bar"))
+      assert(shouldFail.isFailure)
+
+      Cache.url("testprotocol://foo/bar")
+    }
+
+    * - {
+      // the real custom protocol test
+      check(s"${TestprotocolHandler.protocol}://foo/")
+    }
+  }
+
+}