Merge pull request #573 from coursier/topic/metadata-submodule

Get metadata during the tests via specific submodule

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/metadata"]
+	path = tests/metadata
+	url = https://github.com/coursier/test-metadata.git

appveyor.yml

@@ -13,6 +13,7 @@ install:
       }
   - cmd: SET PATH=C:\sbt\sbt\bin;%JAVA_HOME%\bin;%PATH%
   - cmd: SET SBT_OPTS=-XX:MaxPermSize=2g -Xmx4g
+  - git submodule update --init --recursive
   - ps: |
       if (!(Test-Path 'C:\Users\appveyor\.m2\repository\org\anarres\jarjar\jarjar-core\1.0.1-coursier-SNAPSHOT')) {
         iex 'git clone https://github.com/alexarchambault/jarjar'

tests/js/src/test/scala/coursier/test/compatibility/package.scala

@@ -1,8 +1,12 @@
 package coursier.test
 
+import coursier.util.TestEscape
+import coursier.{Fetch, Task}
+
 import scala.concurrent.{Promise, ExecutionContext, Future}
 import scala.scalajs.js
 import js.Dynamic.{global => g}
+import scalaz.{-\/, EitherT, \/-}
 
 package object compatibility {
 
@@ -10,10 +14,10 @@ package object compatibility {
 
   lazy val fs = g.require("fs")
 
-  def textResource(path: String)(implicit ec: ExecutionContext): Future[String] = {
+  private def textResource0(path: String)(implicit ec: ExecutionContext): Future[String] = {
     val p = Promise[String]()
 
-    fs.readFile("tests/shared/src/test/resources/" + path, "utf-8", {
+    fs.readFile(path, "utf-8", {
       (err: js.Dynamic, data: js.Dynamic) =>
         if (js.isUndefined(err) || err == null) p.success(data.asInstanceOf[String])
         else p.failure(new Exception(err.toString))
@@ -23,4 +27,25 @@ package object compatibility {
     p.future
   }
 
+  def textResource(path: String)(implicit ec: ExecutionContext): Future[String] =
+    textResource0("tests/shared/src/test/resources/" + path)
+
+  private val baseRepo = "tests/metadata"
+
+  val artifact: Fetch.Content[Task] = { artifact =>
+    EitherT {
+      assert(artifact.authentication.isEmpty)
+
+      val path = baseRepo + "/" + TestEscape.urlAsPath(artifact.url)
+
+      Task { implicit ec =>
+        textResource0(path)
+          .map(\/-(_))
+          .recoverWith {
+            case e: Exception =>
+              Future.successful(-\/(e.getMessage))
+          }
+      }
+    }
+  }
 }

tests/jvm/src/test/scala/coursier/test/compatibility/package.scala

@@ -1,8 +1,13 @@
 package coursier.test
 
-import coursier.Platform
+import java.io.{FileNotFoundException, InputStream}
+import java.nio.file.{Files, Paths}
+
+import coursier.util.TestEscape
+import coursier.{Cache, Fetch, Platform}
 
 import scala.concurrent.{ExecutionContext, Future}
+import scalaz.{-\/, EitherT, \/, \/-}
 import scalaz.concurrent.Task
 
 package object compatibility {
@@ -22,4 +27,48 @@ package object compatibility {
     new String(Platform.readFullySync(is), "UTF-8")
   }
 
+  private val baseRepo = {
+    val dir = Paths.get("tests/metadata")
+    assert(Files.isDirectory(dir))
+    dir
+  }
+
+  private val fillChunks = sys.env.get("FILL_CHUNKS").exists(s => s == "1" || s == "true")
+
+  val artifact: Fetch.Content[Task] = { artifact =>
+
+    if (artifact.url.startsWith("file:/") || artifact.url.startsWith("http://localhost:"))
+      EitherT(Platform.readFully(
+        Cache.urlConnection(artifact.url, artifact.authentication).getInputStream
+      ))
+    else {
+
+      assert(artifact.authentication.isEmpty)
+
+      val path = baseRepo.resolve(TestEscape.urlAsPath(artifact.url))
+
+      val init = EitherT[Task, String, Unit] {
+        if (Files.exists(path))
+          Task.now(\/-(()))
+        else if (fillChunks)
+          Task[String \/ Unit] {
+            Files.createDirectories(path.getParent)
+            def is() = Cache.urlConnection(artifact.url, artifact.authentication).getInputStream
+            val b = Platform.readFullySync(is())
+            Files.write(path, b)
+            \/-(())
+          }.handle {
+            case e: Exception =>
+              -\/(e.toString)
+          }
+        else
+          Task.now(-\/(s"not found: $path"))
+      }
+
+      init.flatMap { _ =>
+        EitherT(Platform.readFully(Files.newInputStream(path)))
+      }
+    }
+  }
+
 }

tests/metadata

@@ -0,0 +1 @@
+Subproject commit f932032f9cf013a6d7fe205b2a8d53229c926162

tests/shared/src/test/scala/coursier/test/CentralTests.scala

@@ -21,6 +21,10 @@ abstract class CentralTests extends TestSuite {
     MavenRepository(centralBase)
   )
 
+  // different return type on JVM and JS...
+  private def fetch(repositories: Seq[Repository]) =
+    Fetch.from(repositories, compatibility.artifact)
+
   def resolve(
     deps: Set[Dependency],
     filter: Option[Dependency => Boolean] = None,
@@ -29,7 +33,7 @@ abstract class CentralTests extends TestSuite {
   ) = {
     val repositories0 = extraRepo.toSeq ++ repositories
 
-    val fetch = Platform.fetch(repositories0)
+    val fetch0 = fetch(repositories0)
 
     Resolution(
       deps,
@@ -37,7 +41,7 @@ abstract class CentralTests extends TestSuite {
       userActivations = profiles.map(_.iterator.map(_ -> true).toMap)
     )
       .process
-      .run(fetch)
+      .run(fetch0)
       .map { res =>
 
         assert(res.metadataErrors.isEmpty)

tests/shared/src/test/scala/coursier/util/TestEscape.scala

@@ -0,0 +1,53 @@
+package coursier.util
+
+object TestEscape {
+
+  private val unsafeChars: Set[Char] = " %$&+,:;=?@<>#".toSet
+
+  // Scala version of http://stackoverflow.com/questions/4571346/how-to-encode-url-to-avoid-special-characters-in-java/4605848#4605848
+  // '/' was removed from the unsafe character list
+  private def escape(input: String): String = {
+
+    def toHex(ch: Int) =
+      (if (ch < 10) '0' + ch else 'A' + ch - 10).toChar
+
+    def isUnsafe(ch: Char) =
+      ch > 128 || ch < 0 || unsafeChars(ch)
+
+    input.flatMap {
+      case ch if isUnsafe(ch) =>
+        "%" + toHex(ch / 16) + toHex(ch % 16)
+      case other =>
+        other.toString
+    }
+  }
+
+  def urlAsPath(url: String): String = {
+
+    assert(!url.startsWith("file:/"), s"Got file URL: $url")
+
+    url.split(":", 2) match {
+      case Array(protocol, remaining) =>
+        val remaining0 =
+          if (remaining.startsWith("///"))
+            remaining.stripPrefix("///")
+          else if (remaining.startsWith("/"))
+            remaining.stripPrefix("/")
+          else
+            throw new Exception(s"URL $url doesn't contain an absolute path")
+
+        val remaining1 =
+          if (remaining0.endsWith("/"))
+            // keeping directory content in .directory files
+            remaining0 + ".directory"
+          else
+            remaining0
+
+        escape(protocol + "/" + remaining1.dropWhile(_ == '/'))
+
+      case _ =>
+        throw new Exception(s"No protocol found in URL $url")
+    }
+  }
+
+}