Merge pull request #189 from alexarchambault/topic/url-protocol

Add support for custom URL protocols, change the way cache directory is specified
This commit is contained in:
Alexandre Archambault 2016-03-06 19:25:59 +01:00
commit 5d63c9973c
21 changed files with 327 additions and 61 deletions

View File

@ -220,6 +220,13 @@ lazy val cache = project
import com.typesafe.tools.mima.core.ProblemFilters._
Seq(
// Since 1.0.0-M10
// cache argument type changed from `Seq[(String, File)]` to `File`
ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.file"),
ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.fetch"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.Cache.default"),
ProblemFilters.exclude[IncompatibleMethTypeProblem]("coursier.Cache.validateChecksum"),
ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.defaultBase"),
// Since 1.0.0-M9
// Added an optional extra parameter to FileError.NotFound - only
// its unapply method should break compatibility at the source level.

View File

@ -1,7 +1,7 @@
package coursier
import java.math.BigInteger
import java.net.{ HttpURLConnection, URL, URLConnection }
import java.net.{ HttpURLConnection, URL, URLConnection, URLStreamHandler }
import java.nio.channels.{ OverlappingFileLockException, FileLock }
import java.nio.file.{ StandardCopyOption, Files => NioFiles }
import java.security.MessageDigest
@ -43,25 +43,30 @@ object Cache {
}
}
private def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = {
private def withLocal(artifact: Artifact, cache: File): Artifact = {
def local(url: String) =
if (url.startsWith("file:///"))
url.stripPrefix("file://")
else if (url.startsWith("file:/"))
url.stripPrefix("file:")
else {
val localPathOpt = cache.collectFirst {
case (base, cacheDir) if url.startsWith(base) =>
cacheDir.toString + "/" + escape(url.stripPrefix(base))
}
else
// FIXME Should we fully parse the URL here?
// FIXME Should some safeguards be added against '..' components in paths?
url.split(":", 2) match {
case Array(protocol, remaining) =>
val remaining0 =
if (remaining.startsWith("///"))
remaining.stripPrefix("///")
else if (remaining.startsWith("/"))
remaining.stripPrefix("/")
else
throw new Exception(s"URL $url doesn't contain an absolute path")
localPathOpt.getOrElse {
// FIXME Means we were handed an artifact from repositories other than the known ones
println(cache.mkString("\n"))
println(url)
???
new File(cache, escape(protocol + "/" + remaining0)) .toString
case _ =>
throw new Exception(s"No protocol found in URL $url")
}
}
if (artifact.extra.contains("local"))
artifact
@ -178,9 +183,67 @@ object Cache {
private val partialContentResponseCode = 206
private val handlerClsCache = new ConcurrentHashMap[String, Option[URLStreamHandler]]
private def handlerFor(url: String): Option[URLStreamHandler] = {
val protocol = url.takeWhile(_ != ':')
Option(handlerClsCache.get(protocol)) match {
case None =>
val clsName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
val clsOpt =
try Some(Thread.currentThread().getContextClassLoader.loadClass(clsName))
catch {
case _: ClassNotFoundException =>
None
}
def printError(e: Exception): Unit =
scala.Console.err.println(
s"Cannot instantiate $clsName: $e${Option(e.getMessage).map(" ("+_+")")}"
)
val handlerOpt = clsOpt.flatMap {
cls =>
try Some(cls.newInstance().asInstanceOf[URLStreamHandler])
catch {
case e: InstantiationException =>
printError(e)
None
case e: IllegalAccessException =>
printError(e)
None
case e: ClassCastException =>
printError(e)
None
}
}
val prevOpt = Option(handlerClsCache.putIfAbsent(protocol, handlerOpt))
prevOpt.getOrElse(handlerOpt)
case Some(handlerOpt) =>
handlerOpt
}
}
/**
* Returns a `java.net.URL` for `s`, possibly using the custom protocol handlers found under the
* `coursier.cache.protocol` namespace.
*
* E.g. URL `"test://abc.com/foo"`, having protocol `"test"`, can be handled by a
* `URLStreamHandler` named `coursier.cache.protocol.TestHandler` (protocol name gets
* capitalized, and suffixed with `Handler` to get the class name).
*
* @param s
* @return
*/
def url(s: String): URL =
new URL(null, s, handlerFor(s).orNull)
private def download(
artifact: Artifact,
cache: Seq[(String, File)],
cache: File,
checksums: Set[String],
cachePolicy: CachePolicy,
pool: ExecutorService,
@ -213,8 +276,8 @@ object Cache {
.map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
}
def urlConn(url: String) = {
val conn = new URL(url).openConnection() // FIXME Should this be closed?
def urlConn(url0: String) = {
val conn = url(url0).openConnection() // FIXME Should this be closed?
// Dummy user-agent instead of the default "Java/...",
// so that we are not returned incomplete/erroneous metadata
// (Maven 2 compatibility? - happens for snapshot versioning metadata,
@ -460,7 +523,7 @@ object Cache {
def validateChecksum(
artifact: Artifact,
sumType: String,
cache: Seq[(String, File)],
cache: File,
pool: ExecutorService
): EitherT[Task, FileError, Unit] = {
@ -514,7 +577,7 @@ object Cache {
def file(
artifact: Artifact,
cache: Seq[(String, File)] = default,
cache: File = default,
cachePolicy: CachePolicy = CachePolicy.FetchMissing,
checksums: Seq[Option[String]] = defaultChecksums,
logger: Option[Logger] = None,
@ -565,7 +628,7 @@ object Cache {
}
def fetch(
cache: Seq[(String, File)] = default,
cache: File = default,
cachePolicy: CachePolicy = CachePolicy.FetchMissing,
checksums: Seq[Option[String]] = defaultChecksums,
logger: Option[Logger] = None,
@ -613,18 +676,13 @@ object Cache {
dropInfoAttributes = true
)
lazy val defaultBase = new File(
lazy val default = new File(
sys.env.getOrElse(
"COURSIER_CACHE",
sys.props("user.home") + "/.coursier/cache/v1"
)
).getAbsoluteFile
lazy val default = Seq(
"http://" -> new File(defaultBase, "http"),
"https://" -> new File(defaultBase, "https")
)
val defaultConcurrentDownloadCount = 6
lazy val defaultPool =

View File

@ -1,5 +1,7 @@
package coursier
import java.net.MalformedURLException
import coursier.ivy.IvyRepository
import coursier.util.Parse
@ -24,10 +26,13 @@ object CacheParse {
sys.error(s"Unrecognized repository: $r")
}
if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("file:/"))
try {
Cache.url(url)
repo.success
else
s"Unrecognized protocol in $url".failure
} catch {
case e: MalformedURLException =>
("Error parsing URL " + url + Option(e.getMessage).map(" (" + _ + ")").mkString).failure
}
}
def repositories(l: Seq[String]): ValidationNel[String, Seq[Repository]] =

View File

@ -41,7 +41,7 @@ object Platform {
val artifact: Fetch.Content[Task] = { artifact =>
EitherT {
val url = new URL(artifact.url)
val url = Cache.url(artifact.url)
val conn = url.openConnection()
// Dummy user-agent instead of the default "Java/...",

View File

@ -74,7 +74,7 @@ case class CommonOptions(
case class CacheOptions(
@Help("Cache directory (defaults to environment variable COURSIER_CACHE or ~/.coursier/cache/v1)")
@Short("C")
cache: String = Cache.defaultBase.toString
cache: String = Cache.default.toString
)
sealed abstract class CoursierCommand extends Command
@ -490,9 +490,9 @@ case class Bootstrap(
val isolatedUrls = isolatedArtifactFiles.map { case (k, (v, _)) => k -> v }
val isolatedFiles = isolatedArtifactFiles.map { case (k, (_, v)) => k -> v }
val unrecognized = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://"))
if (unrecognized.nonEmpty)
Console.err.println(s"Warning: non HTTP URLs:\n${unrecognized.mkString("\n")}")
val nonHttpUrls = urls.filter(s => !s.startsWith("http://") && !s.startsWith("https://"))
if (nonHttpUrls.nonEmpty)
Console.err.println(s"Warning: non HTTP URLs:\n${nonHttpUrls.mkString("\n")}")
val buffer = new ByteArrayOutputStream()

View File

@ -77,11 +77,7 @@ class Helper(
)
}
val caches =
Seq(
"http://" -> new File(new File(cacheOptions.cache), "http"),
"https://" -> new File(new File(cacheOptions.cache), "https")
)
val cache = new File(cacheOptions.cache)
val pool = Executors.newFixedThreadPool(parallel, Strategy.DefaultDaemonThreadFactory)
@ -200,7 +196,7 @@ class Helper(
None
val fetchs = cachePolicies.map(p =>
Cache.fetch(caches, p, checksums = checksums, logger = logger, pool = pool)
Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool)
)
val fetchQuiet = coursier.Fetch.from(
repositories,
@ -332,8 +328,8 @@ class Helper(
println(s" Found ${artifacts0.length} artifacts")
val tasks = artifacts0.map(artifact =>
(Cache.file(artifact, caches, cachePolicies.head, checksums = checksums, logger = logger, pool = pool) /: cachePolicies.tail)(
_ orElse Cache.file(artifact, caches, _, checksums = checksums, logger = logger, pool = pool)
(Cache.file(artifact, cache, cachePolicies.head, checksums = checksums, logger = logger, pool = pool) /: cachePolicies.tail)(
_ orElse Cache.file(artifact, cache, _, checksums = checksums, logger = logger, pool = pool)
).run.map(artifact.->)
)

View File

@ -39,7 +39,7 @@ object CoursierPlugin extends AutoPlugin {
coursierVerbosity := 1,
coursierResolvers <<= Tasks.coursierResolversTask,
coursierSbtResolvers <<= externalResolvers in updateSbtClassifiers,
coursierCache := Cache.defaultBase,
coursierCache := Cache.default,
update <<= Tasks.updateTask(withClassifiers = false),
updateClassifiers <<= Tasks.updateTask(withClassifiers = true),
updateSbtClassifiers in Defaults.TaskGlobal <<= Tasks.updateTask(withClassifiers = true, sbtClassifiers = true),

View File

@ -1,8 +1,11 @@
package coursier
import coursier.ivy.{ IvyXml, IvyRepository }
import sbt.mavenint.SbtPomExtraProperties
import java.net.MalformedURLException
import sbt.{ Resolver, CrossVersion, ModuleID }
import sbt.mavenint.SbtPomExtraProperties
object FromSbt {
@ -105,12 +108,20 @@ object FromSbt {
def repository(resolver: Resolver, ivyProperties: Map[String, String]): Option[Repository] =
resolver match {
case sbt.MavenRepository(_, root) =>
if (root.startsWith("http://") || root.startsWith("https://") || root.startsWith("file:/")) {
try {
Cache.url(root) // ensure root is a URL whose protocol can be handled here
val root0 = if (root.endsWith("/")) root else root + "/"
Some(MavenRepository(root0, sbtAttrStub = true))
} else {
Console.err.println(s"Warning: unrecognized Maven repository protocol in $root, ignoring it")
None
} catch {
case e: MalformedURLException =>
Console.err.println(
"Warning: error parsing Maven repository base " +
root +
Option(e.getMessage).map(" ("+_+")").mkString +
", ignoring it"
)
None
}
case sbt.FileRepository(_, _, patterns)

View File

@ -215,7 +215,7 @@ object Tasks {
val artifactsChecksums = coursierArtifactsChecksums.value
val maxIterations = coursierMaxIterations.value
val cachePolicy = coursierCachePolicy.value
val cacheDir = coursierCache.value
val cache = coursierCache.value
val sv = scalaVersion.value // is this always defined? (e.g. for Java only projects?)
@ -273,11 +273,6 @@ object Tasks {
val repositories = Seq(globalPluginsRepo, interProjectRepo) ++ resolvers.flatMap(FromSbt.repository(_, ivyProperties))
val caches = Seq(
"http://" -> new File(cacheDir, "http"),
"https://" -> new File(cacheDir, "https")
)
val pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory)
def createLogger() = new TermDisplay(new OutputStreamWriter(System.err))
@ -286,8 +281,8 @@ object Tasks {
val fetch = Fetch.from(
repositories,
Cache.fetch(caches, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool),
Cache.fetch(caches, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool)
Cache.fetch(cache, CachePolicy.LocalOnly, checksums = checksums, logger = Some(resLogger), pool = pool),
Cache.fetch(cache, cachePolicy, checksums = checksums, logger = Some(resLogger), pool = pool)
)
def depsRepr(deps: Seq[(String, Dependency)]) =
@ -411,7 +406,7 @@ object Tasks {
val artifactsLogger = createLogger()
val artifactFileOrErrorTasks = allArtifacts.toVector.map { a =>
Cache.file(a, caches, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _))
Cache.file(a, cache, cachePolicy, checksums = artifactsChecksums, logger = Some(artifactsLogger), pool = pool).run.map((a, _))
}
if (verbosity >= 0)

View File

@ -0,0 +1,83 @@
<?xml version='1.0' encoding='UTF-8'?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<groupId>com.github.alexarchambault</groupId>
<artifactId>coursier_2.11</artifactId>
<packaging>jar</packaging>
<description>coursier</description>
<url>https://github.com/alexarchambault/coursier</url>
<version>1.0.0-M9-test</version>
<licenses>
<license>
<name>Apache 2.0</name>
<url>http://opensource.org/licenses/Apache-2.0</url>
<distribution>repo</distribution>
</license>
</licenses>
<name>coursier</name>
<organization>
<name>com.github.alexarchambault</name>
<url>https://github.com/alexarchambault/coursier</url>
</organization>
<scm>
<connection>scm:git:github.com/alexarchambault/coursier.git</connection>
<developerConnection>scm:git:git@github.com:alexarchambault/coursier.git</developerConnection>
<url>github.com/alexarchambault/coursier.git</url>
</scm>
<developers>
<developer>
<id>alexarchambault</id>
<name>Alexandre Archambault</name>
<url>https://github.com/alexarchambault</url>
</developer>
</developers>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>2.11.7</version>
</dependency>
<dependency>
<groupId>org.scoverage</groupId>
<artifactId>scalac-scoverage-runtime_2.11</artifactId>
<version>1.1.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scoverage</groupId>
<artifactId>scalac-scoverage-plugin_2.11</artifactId>
<version>1.1.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scalaz</groupId>
<artifactId>scalaz-core_2.11</artifactId>
<version>7.1.2</version>
</dependency>
<dependency>
<groupId>org.scala-lang.modules</groupId>
<artifactId>scala-xml_2.11</artifactId>
<version>1.0.3</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>sonatypereleases</id>
<name>sonatype-releases</name>
<url>https://oss.sonatype.org/content/repositories/releases/</url>
<layout>default</layout>
</repository>
<repository>
<id>ScalazBintrayRepo</id>
<name>Scalaz Bintray Repo</name>
<url>http://dl.bintray.com/scalaz/releases/</url>
<layout>default</layout>
</repository>
<repository>
<id>sonatypereleases</id>
<name>sonatype-releases</name>
<url>https://oss.sonatype.org/content/repositories/releases/</url>
<layout>default</layout>
</repository>
</repositories>
</project>

View File

@ -0,0 +1,27 @@
package coursier.cache.protocol
import java.net.{ URL, URLConnection, URLStreamHandler }
class TestprotocolHandler extends URLStreamHandler {
protected def openConnection(url: URL): URLConnection = {
val resPath = "/test-repo/http/abc.com" + url.getPath
val resURLOpt = Option(getClass.getResource(resPath))
resURLOpt match {
case None =>
new URLConnection(url) {
def connect() = throw new NoSuchElementException(s"Resource $resPath")
}
case Some(resURL) =>
resURL.openConnection()
}
}
}
object TestprotocolHandler {
val protocol = "testprotocol"
// get this namespace via a macro?
val expectedClassName = s"coursier.cache.protocol.${protocol.capitalize}Handler"
assert(classOf[TestprotocolHandler].getName == expectedClassName)
}

View File

@ -35,12 +35,9 @@ object ChecksumTests extends TestSuite {
'artifact - {
val cachePath = getClass.getResource("/checksums").getPath
val cachePath = getClass.getResource("/test-repo").getPath
val cache = Seq(
"http://" -> new File(cachePath),
"https://" -> new File(cachePath)
)
val cache = new File(cachePath)
def validate(artifact: Artifact, sumType: String) =
Cache.validateChecksum(

View File

@ -0,0 +1,85 @@
package coursier
package test
import java.io.File
import java.nio.file.Files
import coursier.cache.protocol.TestprotocolHandler
import utest._
import scala.util.Try
object CustomProtocolTests extends TestSuite {
val tests = TestSuite {
def check(extraMavenRepo: String): Unit = {
val tmpDir = Files.createTempDirectory("coursier-protocol-tests").toFile
def cleanTmpDir() = {
def delete(f: File): Boolean =
if (f.isDirectory) {
val removedContent = f.listFiles().map(delete).forall(x => x)
val removedDir = f.delete()
removedContent && removedDir
} else
f.delete()
if (!delete(tmpDir))
Console.err.println(s"Warning: unable to remove temporary directory $tmpDir")
}
val res = try {
val fetch = Fetch.from(
Seq(
MavenRepository(extraMavenRepo),
MavenRepository("https://repo1.maven.org/maven2")
),
Cache.fetch(
tmpDir
)
)
val startRes = Resolution(
Set(
Dependency(
Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test"
)
)
)
startRes.process.run(fetch).run
} finally {
cleanTmpDir()
}
val errors = res.errors
assert(errors.isEmpty)
}
// using scala-test would allow to put the below comments in the test names...
* - {
// test that everything's fine with standard protocols
val repoPath = new File(getClass.getResource("/test-repo/http/abc.com").getPath)
check(repoPath.toURI.toString)
}
* - {
// test the Cache.url method
val shouldFail = Try(Cache.url("notfoundzzzz://foo/bar"))
assert(shouldFail.isFailure)
Cache.url("testprotocol://foo/bar")
}
* - {
// the real custom protocol test
check(s"${TestprotocolHandler.protocol}://foo/")
}
}
}