Merge pull request #265 from alexarchambault/topic/ivy-version-intervals

Add support for version intervals for Ivy repositories
2016-05-30 10:27:18 +02:00 · 2016-05-30 10:27:18 +02:00 · 0a38127d2f
parent c5fb28f982 1553d0b9d9
commit 0a38127d2f
9 changed files with 227 additions and 18 deletions
--- a/build.sbt
+++ b/build.sbt
@ -182,7 +182,8 @@ lazy val core = crossProject
  .jvmSettings(
    libraryDependencies ++=
      Seq(
-        "org.scalaz" %% "scalaz-core" % "7.1.2"
+        "org.scalaz" %% "scalaz-core" % "7.1.2",
+        "org.jsoup" % "jsoup" % "1.9.2"
      ) ++ {
        if (scalaVersion.value.startsWith("2.10.")) Seq()
        else Seq(
@ -531,3 +532,6 @@ lazy val `coursier` = project.in(file("."))
  .settings(commonSettings)
  .settings(noPublishSettings)
  .settings(releaseSettings)
+  .settings(
+    moduleName := "coursier-root"
+  )
--- a/cache/src/main/scala/coursier/Cache.scala
+++ b/cache/src/main/scala/coursier/Cache.scala
@ -68,9 +68,16 @@ object Cache {
              else
                throw new Exception(s"URL $url doesn't contain an absolute path")

+            val remaining1 =
+              if (remaining0.endsWith("/"))
+                // keeping directory content in .directory files
+                remaining0 + ".directory"
+              else
+                remaining0
+
            new File(
              cache,
-              escape(protocol + "/" + user.fold("")(_ + "@") + remaining0.dropWhile(_ == '/'))
+              escape(protocol + "/" + user.fold("")(_ + "@") + remaining1.dropWhile(_ == '/'))
            ).toString

          case _ =>
--- a/core/js/src/main/scala/coursier/core/compatibility/package.scala
+++ b/core/js/src/main/scala/coursier/core/compatibility/package.scala
@ -93,4 +93,9 @@ package object compatibility {
  def encodeURIComponent(s: String): String =
    g.encodeURIComponent(s).asInstanceOf[String]

+  def listWebPageSubDirectories(page: String): Seq[String] = {
+    // TODO
+    ???
+  }
+
 }
--- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala
+++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala
@ -2,8 +2,11 @@ package coursier.core

 import coursier.util.Xml

+import scala.collection.JavaConverters._
 import scala.xml.{ Attribute, MetaData, Null }

+import org.jsoup.Jsoup
+
 package object compatibility {

  implicit class RichChar(val c: Char) extends AnyVal {
@ -53,4 +56,17 @@ package object compatibility {
  def encodeURIComponent(s: String): String =
    new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString

+  def listWebPageSubDirectories(page: String): Seq[String] =
+    Jsoup.parse(page)
+      .select("a[href~=[^/]*/]")
+      .asScala
+      .toVector
+      .map { elem =>
+        elem
+          .attr("href")
+          .stripPrefix(":") // bintray typically prepends these
+          .stripSuffix("/")
+      }
+      .filter(n => n != "." && n != "..")
+
 }
--- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala
+++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala
@ -4,6 +4,7 @@ import coursier.Fetch
 import coursier.core._

 import scalaz._
+import scalaz.Scalaz.ToEitherOps

 case class IvyRepository(
  pattern: String,
@ -25,12 +26,24 @@ case class IvyRepository(
  private val pattern0 = Pattern(pattern, properties)
  private val metadataPattern0 = Pattern(metadataPattern, properties)

+  private val revisionListingPatternOpt = {
+    val idx = metadataPattern.indexOf("[revision]/")
+    if (idx < 0)
+      None
+    else
+      // FIXME A bit too permissive... we should check that [revision] indeed begins
+      // a path component (that is, has a '/' before it no matter what)
+      // This is trickier than simply checking for a '/' character before it in metadataPattern,
+      // because of optional parts in it.
+      Some(Pattern(metadataPattern.take(idx), properties))
+  }
+
  // See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a
  // list of variables that should be supported.
  // Some are missing (branch, conf, originalName).
  private def variables(
    module: Module,
-    version: String,
+    versionOpt: Option[String],
    `type`: String,
    artifact: String,
    ext: String,
@ -41,11 +54,13 @@ case class IvyRepository(
      "organisation" -> module.organization,
      "orgPath" -> module.organization.replace('.', '/'),
      "module" -> module.name,
-      "revision" -> version,
      "type" -> `type`,
      "artifact" -> artifact,
      "ext" -> ext
-    ) ++ module.attributes ++ classifierOpt.map("classifier" -> _).toSeq
+    ) ++
+    module.attributes ++
+    classifierOpt.map("classifier" -> _).toSeq ++
+    versionOpt.map("revision" -> _).toSeq


  val source: Artifact.Source =
@ -79,7 +94,7 @@ case class IvyRepository(
          val retainedWithUrl = retained.flatMap { p =>
            pattern0.substitute(variables(
              dependency.module,
-              dependency.version,
+              Some(project.actualVersion),
              p.`type`,
              p.name,
              p.ext,
@ -118,10 +133,69 @@ case class IvyRepository(
    F: Monad[F]
  ): EitherT[F, String, (Artifact.Source, Project)] = {

+    revisionListingPatternOpt match {
+      case None =>
+        findNoInverval(module, version, fetch)
+      case Some(revisionListingPattern) =>
+        Parse.versionInterval(version)
+          .orElse(Parse.ivyLatestSubRevisionInterval(version))
+          .filter(_.isValid) match {
+          case None =>
+            findNoInverval(module, version, fetch)
+          case Some(itv) =>
+            val listingUrl = revisionListingPattern.substitute(
+              variables(module, None, "ivy", "ivy", "xml", None)
+            ).flatMap { s =>
+              if (s.endsWith("/"))
+                s.right
+              else
+                s"Don't know how to list revisions of $metadataPattern".left
+            }
+
+            def fromWebPage(s: String) = {
+              val subDirs = coursier.core.compatibility.listWebPageSubDirectories(s)
+              val versions = subDirs.map(Parse.version).collect { case Some(v) => v }
+              val versionsInItv = versions.filter(itv.contains)
+
+              if (versionsInItv.isEmpty)
+                EitherT(F.point(s"No version found for $version".left[(Artifact.Source, Project)]))
+              else {
+                val version0 = versionsInItv.max
+                findNoInverval(module, version0.repr, fetch)
+              }
+            }
+
+            def artifactFor(url: String) =
+              Artifact(
+                url,
+                Map.empty,
+                Map.empty,
+                Attributes("", ""),
+                changing = true,
+                authentication
+              )
+
+            for {
+              url <- EitherT(F.point(listingUrl))
+              s <- fetch(artifactFor(url))
+              res <- fromWebPage(s)
+            } yield res
+        }
+    }
+  }
+
+  def findNoInverval[F[_]](
+    module: Module,
+    version: String,
+    fetch: Fetch.Content[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, (Artifact.Source, Project)] = {
+
    val eitherArtifact: String \/ Artifact =
      for {
        url <- metadataPattern0.substitute(
-          variables(module, version, "ivy", "ivy", "xml", None)
+          variables(module, Some(version), "ivy", "ivy", "xml", None)
        )
      } yield {
        var artifact = Artifact(
@ -176,7 +250,9 @@ case class IvyRepository(
        else
          proj0

-      (source, proj)
+      source -> proj.copy(
+        actualVersionOpt = Some(version)
+      )
    }
  }

--- a/tests/jvm/src/test/scala/coursier/test/IvyTests.scala
+++ b/tests/jvm/src/test/scala/coursier/test/IvyTests.scala
@ -9,6 +9,13 @@ object IvyTests extends TestSuite {

  // only tested on the JVM for lack of support of XML attributes in the platform-dependent XML stubs

+  val sbtRepo = IvyRepository(
+    "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/" +
+      "[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)" +
+      "[revision]/[type]s/[artifact](-[classifier]).[ext]",
+    dropInfoAttributes = true
+  )
+
  val tests = TestSuite {
    'dropInfoAttributes - {
      CentralTests.resolutionCheck(
@ -16,17 +23,31 @@ object IvyTests extends TestSuite {
          "org.scala-js", "sbt-scalajs", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10")
        ),
        version = "0.6.6",
-        extraRepo = Some(
-          IvyRepository(
-            "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/" +
-              "[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)" +
-              "[revision]/[type]s/[artifact](-[classifier]).[ext]",
-            dropInfoAttributes = true
-          )
-        ),
+        extraRepo = Some(sbtRepo),
        configuration = "default(compile)"
      )
    }
+
+    'versionIntervals - {
+      // will likely break if new 0.6.x versions are published :-)
+
+      val mod = Module(
+        "com.github.ddispaltro", "sbt-reactjs", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10")
+      )
+      val ver = "0.6.+"
+
+      val expectedArtifactUrl = "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/com.github.ddispaltro/sbt-reactjs/scala_2.10/sbt_0.13/0.6.8/jars/sbt-reactjs.jar"
+
+      * - CentralTests.resolutionCheck(
+        module = mod,
+        version = ver,
+        extraRepo = Some(sbtRepo)
+      )
+
+      * - CentralTests.withArtifact(mod, ver, extraRepo = Some(sbtRepo)) { artifact =>
+        assert(artifact.url == expectedArtifactUrl)
+      }
+    }
  }

 }
--- a/tests/shared/src/test/resources/resolutions/com.github.ddispaltro/sbt-reactjs/sbtVersion_0.13_scalaVersion_2.10/0.6.+
+++ b/tests/shared/src/test/resources/resolutions/com.github.ddispaltro/sbt-reactjs/sbtVersion_0.13_scalaVersion_2.10/0.6.+
@ -0,0 +1,73 @@
+com.fasterxml.jackson.core:jackson-annotations:2.3.0:default
+com.fasterxml.jackson.core:jackson-core:2.3.3:default
+com.fasterxml.jackson.core:jackson-databind:2.3.3:default
+com.github.ddispaltro:sbt-reactjs;sbtVersion=0.13;scalaVersion=2.10:0.6.8:compile
+com.google.code.findbugs:jsr305:1.3.9:default
+com.google.guava:guava:12.0:default
+com.google.protobuf:protobuf-java:2.5.0:default
+com.typesafe:config:1.2.1:default
+com.typesafe:jse_2.10:1.1.2:default
+com.typesafe:npm_2.10:1.1.1:default
+com.typesafe.akka:akka-actor_2.10:2.3.11:default
+com.typesafe.akka:akka-cluster_2.10:2.3.11:default
+com.typesafe.akka:akka-contrib_2.10:2.3.11:default
+com.typesafe.akka:akka-persistence-experimental_2.10:2.3.11:default
+com.typesafe.akka:akka-remote_2.10:2.3.11:default
+com.typesafe.sbt:sbt-js-engine;sbtVersion=0.13;scalaVersion=2.10:1.1.3:compile
+com.typesafe.sbt:sbt-web;sbtVersion=0.13;scalaVersion=2.10:1.2.1:compile
+io.apigee:rhino:1.7R5pre4:default
+io.apigee.trireme:trireme-core:0.8.5:default
+io.apigee.trireme:trireme-node10src:0.8.5:default
+io.netty:netty:3.8.0.Final:default
+io.spray:spray-json_2.10:1.3.2:default
+org.apache.commons:commons-compress:1.9:default
+org.apache.commons:commons-lang3:3.1:default
+org.fusesource.hawtjni:hawtjni-runtime:1.8:default
+org.fusesource.leveldbjni:leveldbjni:1.7:default
+org.fusesource.leveldbjni:leveldbjni-all:1.7:default
+org.fusesource.leveldbjni:leveldbjni-linux32:1.5:default
+org.fusesource.leveldbjni:leveldbjni-linux64:1.5:default
+org.fusesource.leveldbjni:leveldbjni-osx:1.5:default
+org.fusesource.leveldbjni:leveldbjni-win32:1.5:default
+org.fusesource.leveldbjni:leveldbjni-win64:1.5:default
+org.iq80.leveldb:leveldb:0.5:default
+org.iq80.leveldb:leveldb-api:0.5:default
+org.scala-lang:scala-library:2.10.5:default
+org.slf4j:slf4j-api:1.7.12:default
+org.slf4j:slf4j-simple:1.7.12:default
+org.uncommons.maths:uncommons-maths:1.2.2a:default
+org.webjars:amdefine:0.1.0-2:default
+org.webjars:base62js:1.0.0:default
+org.webjars:esprima:13001.1.0-dev-harmony-fb:default
+org.webjars:jstransform:10.1.0:default
+org.webjars:mkdirp:0.5.0:default
+org.webjars:npm:2.11.2:default
+org.webjars:react:0.14.8:default
+org.webjars:source-map:0.1.40-1:default
+org.webjars:webjars-locator:0.25:default
+org.webjars:webjars-locator-core:0.25:default
+org.webjars.npm:amdefine:0.1.0:default
+org.webjars.npm:ast-types:0.8.5:default
+org.webjars.npm:base62:0.1.1:default
+org.webjars.npm:commander:2.5.0:default
+org.webjars.npm:commoner:0.10.3:default
+org.webjars.npm:esprima-fb:15001.1.0-dev-harmony-fb:default
+org.webjars.npm:glob:4.2.1:default
+org.webjars.npm:graceful-fs:3.0.7:default
+org.webjars.npm:iconv-lite:0.4.9:default
+org.webjars.npm:inflight:1.0.4:default
+org.webjars.npm:inherits:2.0.1:default
+org.webjars.npm:install:0.1.8:default
+org.webjars.npm:jstransform:10.1.0:default
+org.webjars.npm:lru-cache:2.7.0:default
+org.webjars.npm:minimatch:1.0.0:default
+org.webjars.npm:minimist:0.0.8:default
+org.webjars.npm:mkdirp:0.5.1:default
+org.webjars.npm:once:1.3.3:default
+org.webjars.npm:private:0.1.6:default
+org.webjars.npm:q:1.1.2:default
+org.webjars.npm:react-tools:0.13.3:default
+org.webjars.npm:recast:0.10.24:default
+org.webjars.npm:sigmund:1.0.1:default
+org.webjars.npm:source-map:0.4.4:default
+org.webjars.npm:wrappy:1.0.1:default
--- a/tests/shared/src/test/resources/resolutions/io.get-coursier/coursier_2.11/1.0.0-SNAPSHOT
+++ b/tests/shared/src/test/resources/resolutions/io.get-coursier/coursier_2.11/1.0.0-SNAPSHOT
@ -1,4 +1,5 @@
 io.get-coursier:coursier_2.11:1.0.0-SNAPSHOT:compile
+org.jsoup:jsoup:1.9.2:default
 org.scala-lang:scala-library:2.11.8:default
 org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4:default
 org.scala-lang.modules:scala-xml_2.11:1.0.4:default
--- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala
+++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala
@ -89,9 +89,15 @@ object CentralTests extends TestSuite {
      assert(result == expected)
    }

-  def withArtifact[T](module: Module, version: String)(f: Artifact => T): Future[T] = async {
+  def withArtifact[T](
+    module: Module,
+    version: String,
+    extraRepo: Option[Repository] = None
+  )(
+    f: Artifact => T
+  ): Future[T] = async {
    val dep = Dependency(module, version, transitive = false)
-    val res = await(resolve(Set(dep)))
+    val res = await(resolve(Set(dep), extraRepo = extraRepo))

    res.artifacts match {
      case Seq(artifact) =>