Add support for custom URL protocols

cache/src/main/scala/coursier/Cache.scala

@@ -1,7 +1,7 @@
 package coursier
 
 import java.math.BigInteger
-import java.net.{ HttpURLConnection, URL, URLConnection }
+import java.net.{ HttpURLConnection, URL, URLConnection, URLStreamHandler }
 import java.nio.channels.{ OverlappingFileLockException, FileLock }
 import java.nio.file.{ StandardCopyOption, Files => NioFiles }
 import java.security.MessageDigest
@@ -183,6 +183,64 @@ object Cache {
 
   private val partialContentResponseCode = 206
 
+  private val handlerClsCache = new ConcurrentHashMap[String, Option[URLStreamHandler]]
+
+  private def handlerFor(url: String): Option[URLStreamHandler] = {
+    val protocol = url.takeWhile(_ != ':')
+
+    Option(handlerClsCache.get(protocol)) match {
+      case None =>
+        val clsName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
+        val clsOpt =
+          try Some(Thread.currentThread().getContextClassLoader.loadClass(clsName))
+          catch {
+            case _: ClassNotFoundException =>
+              None
+          }
+
+        def printError(e: Exception): Unit =
+          scala.Console.err.println(
+            s"Cannot instantiate $clsName: $e${Option(e.getMessage).map(" ("+_+")")}"
+          )
+
+        val handlerOpt = clsOpt.flatMap {
+          cls =>
+            try Some(cls.newInstance().asInstanceOf[URLStreamHandler])
+            catch {
+              case e: InstantiationException =>
+                printError(e)
+                None
+              case e: IllegalAccessException =>
+                printError(e)
+                None
+              case e: ClassCastException =>
+                printError(e)
+                None
+            }
+        }
+
+        val prevOpt = Option(handlerClsCache.putIfAbsent(protocol, handlerOpt))
+        prevOpt.getOrElse(handlerOpt)
+
+      case Some(handlerOpt) =>
+        handlerOpt
+    }
+  }
+
+  /**
+    * Returns a `java.net.URL` for `s`, possibly using the custom protocol handlers found under the
+    * `coursier.cache.protocol` namespace.
+    *
+    * E.g. URL `"test://abc.com/foo"`, having protocol `"test"`, can be handled by a
+    * `URLStreamHandler` named `coursier.cache.protocol.TestHandler` (protocol name gets
+    * capitalized, and suffixed with `Handler` to get the class name).
+    *
+    * @param s
+    * @return
+    */
+  def url(s: String): URL =
+    new URL(null, s, handlerFor(s).orNull)
+
   private def download(
     artifact: Artifact,
     cache: File,
@@ -218,8 +276,8 @@ object Cache {
           .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
       }
 
-    def urlConn(url: String) = {
-      val conn = new URL(url).openConnection() // FIXME Should this be closed?
+    def urlConn(url0: String) = {
+      val conn = url(url0).openConnection() // FIXME Should this be closed?
       // Dummy user-agent instead of the default "Java/...",
       // so that we are not returned incomplete/erroneous metadata
       // (Maven 2 compatibility? - happens for snapshot versioning metadata,

cache/src/main/scala/coursier/CacheParse.scala

@@ -1,5 +1,7 @@
 package coursier
 
+import java.net.MalformedURLException
+
 import coursier.ivy.IvyRepository
 import coursier.util.Parse
 
@@ -24,10 +26,13 @@ object CacheParse {
           sys.error(s"Unrecognized repository: $r")
       }
 
-      if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/"))
+      try {
+        Cache.url(url)
         repo.success
-      else
-        s"Unrecognized protocol in $url".failure
+      } catch {
+        case e: MalformedURLException =>
+          ("Error parsing URL " + url + Option(e.getMessage).map(" (" + _ + ")").mkString).failure
+      }
     }
 
   def repositories(l: Seq[String]): ValidationNel[String, Seq[Repository]] =

cache/src/main/scala/coursier/Platform.scala

@@ -41,7 +41,7 @@ object Platform {
 
   val artifact: Fetch.Content[Task] = { artifact =>
     EitherT {
-      val url = new URL(artifact.url)
+      val url = Cache.url(artifact.url)
 
       val conn = url.openConnection()
       // Dummy user-agent instead of the default "Java/...",

cli/src/main/scala-2.11/coursier/cli/Coursier.scala

@@ -490,9 +490,9 @@ case class Bootstrap(
   val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v }
   val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v }
 
-  val unrecognized = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://"))
-  if (unrecognized.nonEmpty)
-    Console.err.println(s"Warning: non HTTP URLs:\n${unrecognized.mkString("\n")}")
+  val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://"))
+  if (nonHttpUrls.nonEmpty)
+    Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}")
 
   val buffer = new ByteArrayOutputStream()
 

plugin/src/main/scala-2.10/coursier/FromSbt.scala

@@ -1,8 +1,11 @@
 package coursier
 
 import coursier.ivy.{ IvyXml, IvyRepository }
-import sbt.mavenint.SbtPomExtraProperties
+
+import java.net.MalformedURLException
+
 import sbt.{ Resolver, CrossVersion, ModuleID }
+import sbt.mavenint.SbtPomExtraProperties
 
 object FromSbt {
 
@@ -105,12 +108,20 @@ object FromSbt {
   def repository(resolver: Resolver, ivyProperties: Map[String, String]): Option[Repository] =
     resolver match {
       case sbt.MavenRepository(_, root) =>
-        if (root.startsWith("http://") || root.startsWith("https://") || root.startsWith("file:/")) {
+        try {
+          Cache.url(root) // ensure root is a URL whose protocol can be handled here
           val root0 = if (root.endsWith("/")) root else root + "/"
           Some(MavenRepository(root0, sbtAttrStub = true))
-        } else {
-          Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it")
-          None
+        } catch {
+          case e: MalformedURLException =>
+            Console.err.println(
+              "Warning: error parsing Maven repository base " +
+              root +
+              Option(e.getMessage).map(" ("+_+")").mkString +
+              ", ignoring it"
+            )
+
+            None
         }
 
       case sbt.FileRepository(_, _, patterns)

tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom

@@ -0,0 +1,83 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.github.alexarchambault</groupId>
+    <artifactId>coursier_2.11</artifactId>
+    <packaging>jar</packaging>
+    <description>coursier</description>
+    <url>https://github.com/alexarchambault/coursier</url>
+    <version>1.0.0-M9-test</version>
+    <licenses>
+        <license>
+            <name>Apache 2.0</name>
+            <url>http://opensource.org/licenses/Apache-2.0</url>
+            <distribution>repo</distribution>
+        </license>
+    </licenses>
+    <name>coursier</name>
+    <organization>
+        <name>com.github.alexarchambault</name>
+        <url>https://github.com/alexarchambault/coursier</url>
+    </organization>
+    <scm>
+        <connection>scm:git:github.com/alexarchambault/coursier.git</connection>
+        <developerConnection>scm:git:git@github.com:alexarchambault/coursier.git</developerConnection>
+        <url>github.com/alexarchambault/coursier.git</url>
+    </scm>
+    <developers>
+        <developer>
+            <id>alexarchambault</id>
+            <name>Alexandre Archambault</name>
+            <url>https://github.com/alexarchambault</url>
+        </developer>
+    </developers>
+    <dependencies>
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>2.11.7</version>
+        </dependency>
+        <dependency>
+            <groupId>org.scoverage</groupId>
+            <artifactId>scalac-scoverage-runtime_2.11</artifactId>
+            <version>1.1.0</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scoverage</groupId>
+            <artifactId>scalac-scoverage-plugin_2.11</artifactId>
+            <version>1.1.0</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.scalaz</groupId>
+            <artifactId>scalaz-core_2.11</artifactId>
+            <version>7.1.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.scala-lang.modules</groupId>
+            <artifactId>scala-xml_2.11</artifactId>
+            <version>1.0.3</version>
+        </dependency>
+    </dependencies>
+    <repositories>
+        <repository>
+            <id>sonatypereleases</id>
+            <name>sonatype-releases</name>
+            <url>https://oss.sonatype.org/content/repositories/releases/</url>
+            <layout>default</layout>
+        </repository>
+        <repository>
+            <id>ScalazBintrayRepo</id>
+            <name>Scalaz Bintray Repo</name>
+            <url>http://dl.bintray.com/scalaz/releases/</url>
+            <layout>default</layout>
+        </repository>
+        <repository>
+            <id>sonatypereleases</id>
+            <name>sonatype-releases</name>
+            <url>https://oss.sonatype.org/content/repositories/releases/</url>
+            <layout>default</layout>
+        </repository>
+    </repositories>
+</project>

tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.md5

@@ -0,0 +1 @@
+abd72a03c065a31bbe5ede5b16da98a9

tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.sha1

@@ -0,0 +1 @@
+4630461322d079ad7c53c4f2004ed9509ca046c0

tests/jvm/src/test/scala/coursier/cache/protocol/TestprotocolHandler.scala

@@ -0,0 +1,27 @@
+package coursier.cache.protocol
+
+import java.net.{ URL, URLConnection, URLStreamHandler }
+
+class TestprotocolHandler extends URLStreamHandler {
+  protected def openConnection(url: URL): URLConnection = {
+    val resPath = "/test-repo/http/abc.com" + url.getPath
+    val resURLOpt = Option(getClass.getResource(resPath))
+
+    resURLOpt match {
+      case None =>
+        new URLConnection(url) {
+          def connect() = throw new NoSuchElementException(s"Resource $resPath")
+        }
+      case Some(resURL) =>
+        resURL.openConnection()
+    }
+  }
+}
+
+object TestprotocolHandler {
+  val protocol = "testprotocol"
+
+  // get this namespace via a macro?
+  val expectedClassName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
+  assert(classOf[TestprotocolHandler].getName == expectedClassName)
+}

tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala

@@ -35,7 +35,7 @@ object ChecksumTests extends TestSuite {
 
     'artifact - {
 
-      val cachePath = getClass.getResource("/checksums").getPath
+      val cachePath = getClass.getResource("/test-repo").getPath
 
       val cache = new File(cachePath)
 

tests/jvm/src/test/scala/coursier/test/CustomProtocolTests.scala

@@ -0,0 +1,85 @@
+package coursier
+package test
+
+import java.io.File
+import java.nio.file.Files
+
+import coursier.cache.protocol.TestprotocolHandler
+import utest._
+
+import scala.util.Try
+
+object CustomProtocolTests extends TestSuite {
+
+  val tests = TestSuite {
+
+    def check(extraMavenRepo: String): Unit = {
+
+      val tmpDir = Files.createTempDirectory("coursier-protocol-tests").toFile
+
+      def cleanTmpDir() = {
+        def delete(f: File): Boolean =
+          if (f.isDirectory) {
+            val removedContent = f.listFiles().map(delete).forall(x => x)
+            val removedDir = f.delete()
+
+            removedContent && removedDir
+          } else
+            f.delete()
+
+        if (!delete(tmpDir))
+          Console.err.println(s"Warning: unable to remove temporary directory $tmpDir")
+      }
+      
+      val res = try {
+        val fetch = Fetch.from(
+          Seq(
+            MavenRepository(extraMavenRepo),
+            MavenRepository("https://repo1.maven.org/maven2")
+          ),
+          Cache.fetch(
+            tmpDir
+          )
+        )
+
+        val startRes = Resolution(
+          Set(
+            Dependency(
+              Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test"
+            )
+          )
+        )
+
+        startRes.process.run(fetch).run
+      } finally {
+        cleanTmpDir()
+      }
+
+      val errors = res.errors
+
+      assert(errors.isEmpty)
+    }
+
+    // using scala-test would allow to put the below comments in the test names...
+
+    * - {
+      // test that everything's fine with standard protocols
+      val repoPath = new File(getClass.getResource("/test-repo/http/abc.com").getPath)
+      check(repoPath.toURI.toString)
+    }
+
+    * - {
+      // test the Cache.url method
+      val shouldFail = Try(Cache.url("notfoundzzzz://foo/bar"))
+      assert(shouldFail.isFailure)
+
+      Cache.url("testprotocol://foo/bar")
+    }
+
+    * - {
+      // the real custom protocol test
+      check(s"${TestprotocolHandler.protocol}://foo/")
+    }
+  }
+
+}