diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 16a134020..e98068821 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -1,7 +1,7 @@ package coursier import java.math.BigInteger -import java.net.{ HttpURLConnection, URL, URLConnection } +import java.net.{ HttpURLConnection, URL, URLConnection, URLStreamHandler } import java.nio.channels.{ OverlappingFileLockException, FileLock } import java.nio.file.{ StandardCopyOption, Files => NioFiles } import java.security.MessageDigest @@ -183,6 +183,64 @@ object Cache { private val partialContentResponseCode = 206 + private val handlerClsCache = new ConcurrentHashMap[String, Option[URLStreamHandler]] + + private def handlerFor(url: String): Option[URLStreamHandler] = { + val protocol = url.takeWhile(_ != ':') + + Option(handlerClsCache.get(protocol)) match { + case None => + val clsName = s"coursier.cache.protocol.${protocol.capitalize}Handler" + val clsOpt = + try Some(Thread.currentThread().getContextClassLoader.loadClass(clsName)) + catch { + case _: ClassNotFoundException => + None + } + + def printError(e: Exception): Unit = + scala.Console.err.println( + s"Cannot instantiate $clsName: $e${Option(e.getMessage).map(" ("+_+")")}" + ) + + val handlerOpt = clsOpt.flatMap { + cls => + try Some(cls.newInstance().asInstanceOf[URLStreamHandler]) + catch { + case e: InstantiationException => + printError(e) + None + case e: IllegalAccessException => + printError(e) + None + case e: ClassCastException => + printError(e) + None + } + } + + val prevOpt = Option(handlerClsCache.putIfAbsent(protocol, handlerOpt)) + prevOpt.getOrElse(handlerOpt) + + case Some(handlerOpt) => + handlerOpt + } + } + + /** + * Returns a `java.net.URL` for `s`, possibly using the custom protocol handlers found under the + * `coursier.cache.protocol` namespace. + * + * E.g. URL `"test://abc.com/foo"`, having protocol `"test"`, can be handled by a + * `URLStreamHandler` named `coursier.cache.protocol.TestHandler` (protocol name gets + * capitalized, and suffixed with `Handler` to get the class name). + * + * @param s + * @return + */ + def url(s: String): URL = + new URL(null, s, handlerFor(s).orNull) + private def download( artifact: Artifact, cache: File, @@ -218,8 +276,8 @@ object Cache { .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) } - def urlConn(url: String) = { - val conn = new URL(url).openConnection() // FIXME Should this be closed? + def urlConn(url0: String) = { + val conn = url(url0).openConnection() // FIXME Should this be closed? // Dummy user-agent instead of the default "Java/...", // so that we are not returned incomplete/erroneous metadata // (Maven 2 compatibility? - happens for snapshot versioning metadata, diff --git a/cache/src/main/scala/coursier/CacheParse.scala b/cache/src/main/scala/coursier/CacheParse.scala index 8914e3ad4..4add14115 100644 --- a/cache/src/main/scala/coursier/CacheParse.scala +++ b/cache/src/main/scala/coursier/CacheParse.scala @@ -1,5 +1,7 @@ package coursier +import java.net.MalformedURLException + import coursier.ivy.IvyRepository import coursier.util.Parse @@ -24,10 +26,13 @@ object CacheParse { sys.error(s"Unrecognized repository: $r") } - if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/")) + try { + Cache.url(url) repo.success - else - s"Unrecognized protocol in $url".failure + } catch { + case e: MalformedURLException => + ("Error parsing URL " + url + Option(e.getMessage).map(" (" + _ + ")").mkString).failure + } } def repositories(l: Seq[String]): ValidationNel[String, Seq[Repository]] = diff --git a/cache/src/main/scala/coursier/Platform.scala b/cache/src/main/scala/coursier/Platform.scala index 8eaad94f6..2e7d7b6d3 100644 --- a/cache/src/main/scala/coursier/Platform.scala +++ b/cache/src/main/scala/coursier/Platform.scala @@ -41,7 +41,7 @@ object Platform { val artifact: Fetch.Content[Task] = { artifact => EitherT { - val url = new URL(artifact.url) + val url = Cache.url(artifact.url) val conn = url.openConnection() // Dummy user-agent instead of the default "Java/...", diff --git a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala index acb715e94..8e4e2d85b 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Coursier.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Coursier.scala @@ -490,9 +490,9 @@ case class Bootstrap( val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v } val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v } - val unrecognized = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://")) - if (unrecognized.nonEmpty) - Console.err.println(s"Warning: non HTTP URLs:\n${unrecognized.mkString("\n")}") + val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://")) + if (nonHttpUrls.nonEmpty) + Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}") val buffer = new ByteArrayOutputStream() diff --git a/plugin/src/main/scala-2.10/coursier/FromSbt.scala b/plugin/src/main/scala-2.10/coursier/FromSbt.scala index a7740b1e4..65c24f7b9 100644 --- a/plugin/src/main/scala-2.10/coursier/FromSbt.scala +++ b/plugin/src/main/scala-2.10/coursier/FromSbt.scala @@ -1,8 +1,11 @@ package coursier import coursier.ivy.{ IvyXml, IvyRepository } -import sbt.mavenint.SbtPomExtraProperties + +import java.net.MalformedURLException + import sbt.{ Resolver, CrossVersion, ModuleID } +import sbt.mavenint.SbtPomExtraProperties object FromSbt { @@ -105,12 +108,20 @@ object FromSbt { def repository(resolver: Resolver, ivyProperties: Map[String, String]): Option[Repository] = resolver match { case sbt.MavenRepository(_, root) => - if (root.startsWith("http://") || root.startsWith("https://") || root.startsWith("file:/")) { + try { + Cache.url(root) // ensure root is a URL whose protocol can be handled here val root0 = if (root.endsWith("/")) root else root + "/" Some(MavenRepository(root0, sbtAttrStub = true)) - } else { - Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it") - None + } catch { + case e: MalformedURLException => + Console.err.println( + "Warning: error parsing Maven repository base " + + root + + Option(e.getMessage).map(" ("+_+")").mkString + + ", ignoring it" + ) + + None } case sbt.FileRepository(_, _, patterns) diff --git a/tests/jvm/src/test/resources/checksums/abc.com/com/abc/test/0.1/test-0.1.pom b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.pom similarity index 100% rename from tests/jvm/src/test/resources/checksums/abc.com/com/abc/test/0.1/test-0.1.pom rename to tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.pom diff --git a/tests/jvm/src/test/resources/checksums/abc.com/com/abc/test/0.1/test-0.1.pom.md5 b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.pom.md5 similarity index 100% rename from tests/jvm/src/test/resources/checksums/abc.com/com/abc/test/0.1/test-0.1.pom.md5 rename to tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.pom.md5 diff --git a/tests/jvm/src/test/resources/checksums/abc.com/com/abc/test/0.1/test-0.1.pom.sha1 b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.pom.sha1 similarity index 100% rename from tests/jvm/src/test/resources/checksums/abc.com/com/abc/test/0.1/test-0.1.pom.sha1 rename to tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.pom.sha1 diff --git a/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom new file mode 100644 index 000000000..725552d65 --- /dev/null +++ b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom @@ -0,0 +1,83 @@ + + + 4.0.0 + com.github.alexarchambault + coursier_2.11 + jar + coursier + https://github.com/alexarchambault/coursier + 1.0.0-M9-test + + + Apache 2.0 + http://opensource.org/licenses/Apache-2.0 + repo + + + coursier + + com.github.alexarchambault + https://github.com/alexarchambault/coursier + + + scm:git:github.com/alexarchambault/coursier.git + scm:git:git@github.com:alexarchambault/coursier.git + github.com/alexarchambault/coursier.git + + + + alexarchambault + Alexandre Archambault + https://github.com/alexarchambault + + + + + org.scala-lang + scala-library + 2.11.7 + + + org.scoverage + scalac-scoverage-runtime_2.11 + 1.1.0 + provided + + + org.scoverage + scalac-scoverage-plugin_2.11 + 1.1.0 + provided + + + org.scalaz + scalaz-core_2.11 + 7.1.2 + + + org.scala-lang.modules + scala-xml_2.11 + 1.0.3 + + + + + sonatypereleases + sonatype-releases + https://oss.sonatype.org/content/repositories/releases/ + default + + + ScalazBintrayRepo + Scalaz Bintray Repo + http://dl.bintray.com/scalaz/releases/ + default + + + sonatypereleases + sonatype-releases + https://oss.sonatype.org/content/repositories/releases/ + default + + + \ No newline at end of file diff --git a/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.md5 b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.md5 new file mode 100644 index 000000000..ce80f9a4c --- /dev/null +++ b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.md5 @@ -0,0 +1 @@ +abd72a03c065a31bbe5ede5b16da98a9 \ No newline at end of file diff --git a/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.sha1 b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.sha1 new file mode 100644 index 000000000..0c610868d --- /dev/null +++ b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9-test/coursier_2.11-1.0.0-M9-test.pom.sha1 @@ -0,0 +1 @@ +4630461322d079ad7c53c4f2004ed9509ca046c0 \ No newline at end of file diff --git a/tests/jvm/src/test/resources/checksums/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom similarity index 100% rename from tests/jvm/src/test/resources/checksums/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom rename to tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom diff --git a/tests/jvm/src/test/resources/checksums/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.md5 b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.md5 similarity index 100% rename from tests/jvm/src/test/resources/checksums/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.md5 rename to tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.md5 diff --git a/tests/jvm/src/test/resources/checksums/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.sha1 b/tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.sha1 similarity index 100% rename from tests/jvm/src/test/resources/checksums/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.sha1 rename to tests/jvm/src/test/resources/test-repo/http/abc.com/com/github/alexarchambault/coursier_2.11/1.0.0-M9/coursier_2.11-1.0.0-M9.pom.sha1 diff --git a/tests/jvm/src/test/scala/coursier/cache/protocol/TestprotocolHandler.scala b/tests/jvm/src/test/scala/coursier/cache/protocol/TestprotocolHandler.scala new file mode 100644 index 000000000..7f5094dfc --- /dev/null +++ b/tests/jvm/src/test/scala/coursier/cache/protocol/TestprotocolHandler.scala @@ -0,0 +1,27 @@ +package coursier.cache.protocol + +import java.net.{ URL, URLConnection, URLStreamHandler } + +class TestprotocolHandler extends URLStreamHandler { + protected def openConnection(url: URL): URLConnection = { + val resPath = "/test-repo/http/abc.com" + url.getPath + val resURLOpt = Option(getClass.getResource(resPath)) + + resURLOpt match { + case None => + new URLConnection(url) { + def connect() = throw new NoSuchElementException(s"Resource $resPath") + } + case Some(resURL) => + resURL.openConnection() + } + } +} + +object TestprotocolHandler { + val protocol = "testprotocol" + + // get this namespace via a macro? + val expectedClassName = s"coursier.cache.protocol.${protocol.capitalize}Handler" + assert(classOf[TestprotocolHandler].getName == expectedClassName) +} \ No newline at end of file diff --git a/tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala b/tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala index a08755b03..5ab7b3b36 100644 --- a/tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/ChecksumTests.scala @@ -35,7 +35,7 @@ object ChecksumTests extends TestSuite { 'artifact - { - val cachePath = getClass.getResource("/checksums").getPath + val cachePath = getClass.getResource("/test-repo").getPath val cache = new File(cachePath) diff --git a/tests/jvm/src/test/scala/coursier/test/CustomProtocolTests.scala b/tests/jvm/src/test/scala/coursier/test/CustomProtocolTests.scala new file mode 100644 index 000000000..08597d8d9 --- /dev/null +++ b/tests/jvm/src/test/scala/coursier/test/CustomProtocolTests.scala @@ -0,0 +1,85 @@ +package coursier +package test + +import java.io.File +import java.nio.file.Files + +import coursier.cache.protocol.TestprotocolHandler +import utest._ + +import scala.util.Try + +object CustomProtocolTests extends TestSuite { + + val tests = TestSuite { + + def check(extraMavenRepo: String): Unit = { + + val tmpDir = Files.createTempDirectory("coursier-protocol-tests").toFile + + def cleanTmpDir() = { + def delete(f: File): Boolean = + if (f.isDirectory) { + val removedContent = f.listFiles().map(delete).forall(x => x) + val removedDir = f.delete() + + removedContent && removedDir + } else + f.delete() + + if (!delete(tmpDir)) + Console.err.println(s"Warning: unable to remove temporary directory $tmpDir") + } + + val res = try { + val fetch = Fetch.from( + Seq( + MavenRepository(extraMavenRepo), + MavenRepository("https://repo1.maven.org/maven2") + ), + Cache.fetch( + tmpDir + ) + ) + + val startRes = Resolution( + Set( + Dependency( + Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test" + ) + ) + ) + + startRes.process.run(fetch).run + } finally { + cleanTmpDir() + } + + val errors = res.errors + + assert(errors.isEmpty) + } + + // using scala-test would allow to put the below comments in the test names... + + * - { + // test that everything's fine with standard protocols + val repoPath = new File(getClass.getResource("/test-repo/http/abc.com").getPath) + check(repoPath.toURI.toString) + } + + * - { + // test the Cache.url method + val shouldFail = Try(Cache.url("notfoundzzzz://foo/bar")) + assert(shouldFail.isFailure) + + Cache.url("testprotocol://foo/bar") + } + + * - { + // the real custom protocol test + check(s"${TestprotocolHandler.protocol}://foo/") + } + } + +}