Add support for Ivy version ranges

build.sbt

@@ -182,7 +182,8 @@ lazy val core = crossProject
   .jvmSettings(
     libraryDependencies ++=
       Seq(
-        "org.scalaz" %% "scalaz-core" % "7.1.2"
+        "org.scalaz" %% "scalaz-core" % "7.1.2",
+        "org.jsoup" % "jsoup" % "1.9.2"
       ) ++ {
         if (scalaVersion.value.startsWith("2.10.")) Seq()
         else Seq(

cache/src/main/scala/coursier/Cache.scala

@@ -68,9 +68,16 @@ object Cache {
               else
                 throw new Exception(s"URL $url doesn't contain an absolute path")
 
+            val remaining1 =
+              if (remaining0.endsWith("/"))
+                // keeping directory content in .directory files
+                remaining0 + ".directory"
+              else
+                remaining0
+
             new File(
               cache,
-              escape(protocol + "/" + user.fold("")(_ + "@") + remaining0.dropWhile(_ == '/'))
+              escape(protocol + "/" + user.fold("")(_ + "@") + remaining1.dropWhile(_ == '/'))
             ).toString
 
           case _ =>

core/js/src/main/scala/coursier/core/compatibility/package.scala

@@ -93,4 +93,9 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     g.encodeURIComponent(s).asInstanceOf[String]
 
+  def listWebPageSubDirectories(page: String): Seq[String] = {
+    // TODO
+    ???
+  }
+
 }

core/jvm/src/main/scala/coursier/core/compatibility/package.scala

@@ -2,8 +2,11 @@ package coursier.core
 
 import coursier.util.Xml
 
+import scala.collection.JavaConverters._
 import scala.xml.{ Attribute, MetaData, Null }
 
+import org.jsoup.Jsoup
+
 package object compatibility {
 
   implicit class RichChar(val c: Char) extends AnyVal {
@@ -53,4 +56,17 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString
 
+  def listWebPageSubDirectories(page: String): Seq[String] =
+    Jsoup.parse(page)
+      .select("a[href~=[^/]*/]")
+      .asScala
+      .toVector
+      .map { elem =>
+        elem
+          .attr("href")
+          .stripPrefix(":") // bintray typically prepends these
+          .stripSuffix("/")
+      }
+      .filter(n => n != "." && n != "..")
+
 }

core/shared/src/main/scala/coursier/ivy/IvyRepository.scala

@@ -4,6 +4,7 @@ import coursier.Fetch
 import coursier.core._
 
 import scalaz._
+import scalaz.Scalaz.ToEitherOps
 
 case class IvyRepository(
   pattern: String,
@@ -25,12 +26,24 @@ case class IvyRepository(
   private val pattern0 = Pattern(pattern, properties)
   private val metadataPattern0 = Pattern(metadataPattern, properties)
 
+  private val revisionListingPatternOpt = {
+    val idx = metadataPattern.indexOf("[revision]/")
+    if (idx < 0)
+      None
+    else
+      // FIXME A bit too permissive... we should check that [revision] indeed begins
+      // a path component (that is, has a '/' before it no matter what)
+      // This is trickier than simply checking for a '/' character before it in metadataPattern,
+      // because of optional parts in it.
+      Some(Pattern(metadataPattern.take(idx), properties))
+  }
+
   // See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a
   // list of variables that should be supported.
   // Some are missing (branch, conf, originalName).
   private def variables(
     module: Module,
-    version: String,
+    versionOpt: Option[String],
     `type`: String,
     artifact: String,
     ext: String,
@@ -41,11 +54,13 @@ case class IvyRepository(
       "organisation" -> module.organization,
       "orgPath" -> module.organization.replace('.', '/'),
       "module" -> module.name,
-      "revision" -> version,
       "type" -> `type`,
       "artifact" -> artifact,
       "ext" -> ext
-    ) ++ module.attributes ++ classifierOpt.map("classifier" -> _).toSeq
+    ) ++
+    module.attributes ++
+    classifierOpt.map("classifier" -> _).toSeq ++
+    versionOpt.map("revision" -> _).toSeq
 
 
   val source: Artifact.Source =
@@ -79,7 +94,7 @@ case class IvyRepository(
           val retainedWithUrl = retained.flatMap { p =>
             pattern0.substitute(variables(
               dependency.module,
-              dependency.version,
+              Some(project.actualVersion),
               p.`type`,
               p.name,
               p.ext,
@@ -118,10 +133,69 @@ case class IvyRepository(
     F: Monad[F]
   ): EitherT[F, String, (Artifact.Source, Project)] = {
 
+    revisionListingPatternOpt match {
+      case None =>
+        findNoInverval(module, version, fetch)
+      case Some(revisionListingPattern) =>
+        Parse.versionInterval(version)
+          .orElse(Parse.ivyLatestSubRevisionInterval(version))
+          .filter(_.isValid) match {
+          case None =>
+            findNoInverval(module, version, fetch)
+          case Some(itv) =>
+            val listingUrl = revisionListingPattern.substitute(
+              variables(module, None, "ivy", "ivy", "xml", None)
+            ).flatMap { s =>
+              if (s.endsWith("/"))
+                s.right
+              else
+                s"Don't know how to list revisions of $metadataPattern".left
+            }
+
+            def fromWebPage(s: String) = {
+              val subDirs = coursier.core.compatibility.listWebPageSubDirectories(s)
+              val versions = subDirs.map(Parse.version).collect { case Some(v) => v }
+              val versionsInItv = versions.filter(itv.contains)
+
+              if (versionsInItv.isEmpty)
+                EitherT(F.point(s"No version found for $version".left[(Artifact.Source, Project)]))
+              else {
+                val version0 = versionsInItv.max
+                findNoInverval(module, version0.repr, fetch)
+              }
+            }
+
+            def artifactFor(url: String) =
+              Artifact(
+                url,
+                Map.empty,
+                Map.empty,
+                Attributes("", ""),
+                changing = true,
+                authentication
+              )
+
+            for {
+              url <- EitherT(F.point(listingUrl))
+              s <- fetch(artifactFor(url))
+              res <- fromWebPage(s)
+            } yield res
+        }
+    }
+  }
+
+  def findNoInverval[F[_]](
+    module: Module,
+    version: String,
+    fetch: Fetch.Content[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, (Artifact.Source, Project)] = {
+
     val eitherArtifact: String \/ Artifact =
       for {
         url <- metadataPattern0.substitute(
-          variables(module, version, "ivy", "ivy", "xml", None)
+          variables(module, Some(version), "ivy", "ivy", "xml", None)
         )
       } yield {
         var artifact = Artifact(
@@ -176,7 +250,9 @@ case class IvyRepository(
         else
           proj0
 
-      (source, proj)
+      source -> proj.copy(
+        actualVersionOpt = Some(version)
+      )
     }
   }
 

tests/jvm/src/test/scala/coursier/test/IvyTests.scala

@@ -9,6 +9,13 @@ object IvyTests extends TestSuite {
 
   // only tested on the JVM for lack of support of XML attributes in the platform-dependent XML stubs
 
+  val sbtRepo = IvyRepository(
+    "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/" +
+      "[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)" +
+      "[revision]/[type]s/[artifact](-[classifier]).[ext]",
+    dropInfoAttributes = true
+  )
+
   val tests = TestSuite {
     'dropInfoAttributes - {
       CentralTests.resolutionCheck(
@@ -16,17 +23,31 @@ object IvyTests extends TestSuite {
           "org.scala-js", "sbt-scalajs", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10")
         ),
         version = "0.6.6",
-        extraRepo = Some(
+        extraRepo = Some(sbtRepo),
-          IvyRepository(
-            "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/" +
-              "[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)" +
-              "[revision]/[type]s/[artifact](-[classifier]).[ext]",
-            dropInfoAttributes = true
-          )
-        ),
         configuration = "default(compile)"
       )
     }
+
+    'versionIntervals - {
+      // will likely break if new 0.6.x versions are published :-)
+
+      val mod = Module(
+        "com.github.ddispaltro", "sbt-reactjs", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10")
+      )
+      val ver = "0.6.+"
+
+      val expectedArtifactUrl = "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/com.github.ddispaltro/sbt-reactjs/scala_2.10/sbt_0.13/0.6.8/jars/sbt-reactjs.jar"
+
+      * - CentralTests.resolutionCheck(
+        module = mod,
+        version = ver,
+        extraRepo = Some(sbtRepo)
+      )
+
+      * - CentralTests.withArtifact(mod, ver, extraRepo = Some(sbtRepo)) { artifact =>
+        assert(artifact.url == expectedArtifactUrl)
+      }
+    }
   }
 
 }

tests/shared/src/test/resources/resolutions/com.github.ddispaltro/sbt-reactjs/sbtVersion_0.13_scalaVersion_2.10/0.6.+

@@ -0,0 +1,73 @@
+com.fasterxml.jackson.core:jackson-annotations:2.3.0:default
+com.fasterxml.jackson.core:jackson-core:2.3.3:default
+com.fasterxml.jackson.core:jackson-databind:2.3.3:default
+com.github.ddispaltro:sbt-reactjs;sbtVersion=0.13;scalaVersion=2.10:0.6.8:compile
+com.google.code.findbugs:jsr305:1.3.9:default
+com.google.guava:guava:12.0:default
+com.google.protobuf:protobuf-java:2.5.0:default
+com.typesafe:config:1.2.1:default
+com.typesafe:jse_2.10:1.1.2:default
+com.typesafe:npm_2.10:1.1.1:default
+com.typesafe.akka:akka-actor_2.10:2.3.11:default
+com.typesafe.akka:akka-cluster_2.10:2.3.11:default
+com.typesafe.akka:akka-contrib_2.10:2.3.11:default
+com.typesafe.akka:akka-persistence-experimental_2.10:2.3.11:default
+com.typesafe.akka:akka-remote_2.10:2.3.11:default
+com.typesafe.sbt:sbt-js-engine;sbtVersion=0.13;scalaVersion=2.10:1.1.3:compile
+com.typesafe.sbt:sbt-web;sbtVersion=0.13;scalaVersion=2.10:1.2.1:compile
+io.apigee:rhino:1.7R5pre4:default
+io.apigee.trireme:trireme-core:0.8.5:default
+io.apigee.trireme:trireme-node10src:0.8.5:default
+io.netty:netty:3.8.0.Final:default
+io.spray:spray-json_2.10:1.3.2:default
+org.apache.commons:commons-compress:1.9:default
+org.apache.commons:commons-lang3:3.1:default
+org.fusesource.hawtjni:hawtjni-runtime:1.8:default
+org.fusesource.leveldbjni:leveldbjni:1.7:default
+org.fusesource.leveldbjni:leveldbjni-all:1.7:default
+org.fusesource.leveldbjni:leveldbjni-linux32:1.5:default
+org.fusesource.leveldbjni:leveldbjni-linux64:1.5:default
+org.fusesource.leveldbjni:leveldbjni-osx:1.5:default
+org.fusesource.leveldbjni:leveldbjni-win32:1.5:default
+org.fusesource.leveldbjni:leveldbjni-win64:1.5:default
+org.iq80.leveldb:leveldb:0.5:default
+org.iq80.leveldb:leveldb-api:0.5:default
+org.scala-lang:scala-library:2.10.5:default
+org.slf4j:slf4j-api:1.7.12:default
+org.slf4j:slf4j-simple:1.7.12:default
+org.uncommons.maths:uncommons-maths:1.2.2a:default
+org.webjars:amdefine:0.1.0-2:default
+org.webjars:base62js:1.0.0:default
+org.webjars:esprima:13001.1.0-dev-harmony-fb:default
+org.webjars:jstransform:10.1.0:default
+org.webjars:mkdirp:0.5.0:default
+org.webjars:npm:2.11.2:default
+org.webjars:react:0.14.8:default
+org.webjars:source-map:0.1.40-1:default
+org.webjars:webjars-locator:0.25:default
+org.webjars:webjars-locator-core:0.25:default
+org.webjars.npm:amdefine:0.1.0:default
+org.webjars.npm:ast-types:0.8.5:default
+org.webjars.npm:base62:0.1.1:default
+org.webjars.npm:commander:2.5.0:default
+org.webjars.npm:commoner:0.10.3:default
+org.webjars.npm:esprima-fb:15001.1.0-dev-harmony-fb:default
+org.webjars.npm:glob:4.2.1:default
+org.webjars.npm:graceful-fs:3.0.7:default
+org.webjars.npm:iconv-lite:0.4.9:default
+org.webjars.npm:inflight:1.0.4:default
+org.webjars.npm:inherits:2.0.1:default
+org.webjars.npm:install:0.1.8:default
+org.webjars.npm:jstransform:10.1.0:default
+org.webjars.npm:lru-cache:2.7.0:default
+org.webjars.npm:minimatch:1.0.0:default
+org.webjars.npm:minimist:0.0.8:default
+org.webjars.npm:mkdirp:0.5.1:default
+org.webjars.npm:once:1.3.3:default
+org.webjars.npm:private:0.1.6:default
+org.webjars.npm:q:1.1.2:default
+org.webjars.npm:react-tools:0.13.3:default
+org.webjars.npm:recast:0.10.24:default
+org.webjars.npm:sigmund:1.0.1:default
+org.webjars.npm:source-map:0.4.4:default
+org.webjars.npm:wrappy:1.0.1:default

tests/shared/src/test/resources/resolutions/io.get-coursier/coursier_2.11/1.0.0-SNAPSHOT

@@ -1,4 +1,5 @@
 io.get-coursier:coursier_2.11:1.0.0-SNAPSHOT:compile
+org.jsoup:jsoup:1.9.2:default
 org.scala-lang:scala-library:2.11.8:default
 org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4:default
 org.scala-lang.modules:scala-xml_2.11:1.0.4:default

tests/shared/src/test/scala/coursier/test/CentralTests.scala

@@ -89,9 +89,15 @@ object CentralTests extends TestSuite {
       assert(result == expected)
     }
 
-  def withArtifact[T](module: Module, version: String)(f: Artifact => T): Future[T] = async {
+  def withArtifact[T](
+    module: Module,
+    version: String,
+    extraRepo: Option[Repository] = None
+  )(
+    f: Artifact => T
+  ): Future[T] = async {
     val dep = Dependency(module, version, transitive = false)
-    val res = await(resolve(Set(dep)))
+    val res = await(resolve(Set(dep), extraRepo = extraRepo))
 
     res.artifacts match {
       case Seq(artifact) =>