From 24235c12cc523edeb8cc9dd8ca4ec29f15a81782 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 29 May 2016 23:45:55 +0200 Subject: [PATCH 1/2] Change module name of root project not to confuse sbt-coursier --- build.sbt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sbt b/build.sbt index 2351dfee6..6faf66daf 100644 --- a/build.sbt +++ b/build.sbt @@ -531,3 +531,6 @@ lazy val `coursier` = project.in(file(".")) .settings(commonSettings) .settings(noPublishSettings) .settings(releaseSettings) + .settings( + moduleName := "coursier-root" + ) From 1553d0b9d9cd72657adf847548cccb82ba58798d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sun, 29 May 2016 23:45:58 +0200 Subject: [PATCH 2/2] Add support for Ivy version ranges --- build.sbt | 3 +- cache/src/main/scala/coursier/Cache.scala | 9 +- .../coursier/core/compatibility/package.scala | 5 ++ .../coursier/core/compatibility/package.scala | 16 ++++ .../scala/coursier/ivy/IvyRepository.scala | 88 +++++++++++++++++-- .../test/scala/coursier/test/IvyTests.scala | 37 ++++++-- .../sbtVersion_0.13_scalaVersion_2.10/0.6.+ | 73 +++++++++++++++ .../coursier_2.11/1.0.0-SNAPSHOT | 1 + .../scala/coursier/test/CentralTests.scala | 10 ++- 9 files changed, 224 insertions(+), 18 deletions(-) create mode 100644 tests/shared/src/test/resources/resolutions/com.github.ddispaltro/sbt-reactjs/sbtVersion_0.13_scalaVersion_2.10/0.6.+ diff --git a/build.sbt b/build.sbt index 6faf66daf..5a3e39a4e 100644 --- a/build.sbt +++ b/build.sbt @@ -182,7 +182,8 @@ lazy val core = crossProject .jvmSettings( libraryDependencies ++= Seq( - "org.scalaz" %% "scalaz-core" % "7.1.2" + "org.scalaz" %% "scalaz-core" % "7.1.2", + "org.jsoup" % "jsoup" % "1.9.2" ) ++ { if (scalaVersion.value.startsWith("2.10.")) Seq() else Seq( diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 6226f1d03..f0f651749 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -68,9 +68,16 @@ object Cache { else throw new Exception(s"URL $url doesn't contain an absolute path") + val remaining1 = + if (remaining0.endsWith("/")) + // keeping directory content in .directory files + remaining0 + ".directory" + else + remaining0 + new File( cache, - escape(protocol + "/" + user.fold("")(_ + "@") + remaining0.dropWhile(_ == '/')) + escape(protocol + "/" + user.fold("")(_ + "@") + remaining1.dropWhile(_ == '/')) ).toString case _ => diff --git a/core/js/src/main/scala/coursier/core/compatibility/package.scala b/core/js/src/main/scala/coursier/core/compatibility/package.scala index b168d85e1..eebc41fa1 100644 --- a/core/js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/js/src/main/scala/coursier/core/compatibility/package.scala @@ -93,4 +93,9 @@ package object compatibility { def encodeURIComponent(s: String): String = g.encodeURIComponent(s).asInstanceOf[String] + def listWebPageSubDirectories(page: String): Seq[String] = { + // TODO + ??? + } + } diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index c290caaa5..11f052fe2 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -2,8 +2,11 @@ package coursier.core import coursier.util.Xml +import scala.collection.JavaConverters._ import scala.xml.{ Attribute, MetaData, Null } +import org.jsoup.Jsoup + package object compatibility { implicit class RichChar(val c: Char) extends AnyVal { @@ -53,4 +56,17 @@ package object compatibility { def encodeURIComponent(s: String): String = new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString + def listWebPageSubDirectories(page: String): Seq[String] = + Jsoup.parse(page) + .select("a[href~=[^/]*/]") + .asScala + .toVector + .map { elem => + elem + .attr("href") + .stripPrefix(":") // bintray typically prepends these + .stripSuffix("/") + } + .filter(n => n != "." && n != "..") + } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 8df652abd..f309ad872 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -4,6 +4,7 @@ import coursier.Fetch import coursier.core._ import scalaz._ +import scalaz.Scalaz.ToEitherOps case class IvyRepository( pattern: String, @@ -25,12 +26,24 @@ case class IvyRepository( private val pattern0 = Pattern(pattern, properties) private val metadataPattern0 = Pattern(metadataPattern, properties) + private val revisionListingPatternOpt = { + val idx = metadataPattern.indexOf("[revision]/") + if (idx < 0) + None + else + // FIXME A bit too permissive... we should check that [revision] indeed begins + // a path component (that is, has a '/' before it no matter what) + // This is trickier than simply checking for a '/' character before it in metadataPattern, + // because of optional parts in it. + Some(Pattern(metadataPattern.take(idx), properties)) + } + // See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a // list of variables that should be supported. // Some are missing (branch, conf, originalName). private def variables( module: Module, - version: String, + versionOpt: Option[String], `type`: String, artifact: String, ext: String, @@ -41,11 +54,13 @@ case class IvyRepository( "organisation" -> module.organization, "orgPath" -> module.organization.replace('.', '/'), "module" -> module.name, - "revision" -> version, "type" -> `type`, "artifact" -> artifact, "ext" -> ext - ) ++ module.attributes ++ classifierOpt.map("classifier" -> _).toSeq + ) ++ + module.attributes ++ + classifierOpt.map("classifier" -> _).toSeq ++ + versionOpt.map("revision" -> _).toSeq val source: Artifact.Source = @@ -79,7 +94,7 @@ case class IvyRepository( val retainedWithUrl = retained.flatMap { p => pattern0.substitute(variables( dependency.module, - dependency.version, + Some(project.actualVersion), p.`type`, p.name, p.ext, @@ -118,10 +133,69 @@ case class IvyRepository( F: Monad[F] ): EitherT[F, String, (Artifact.Source, Project)] = { + revisionListingPatternOpt match { + case None => + findNoInverval(module, version, fetch) + case Some(revisionListingPattern) => + Parse.versionInterval(version) + .orElse(Parse.ivyLatestSubRevisionInterval(version)) + .filter(_.isValid) match { + case None => + findNoInverval(module, version, fetch) + case Some(itv) => + val listingUrl = revisionListingPattern.substitute( + variables(module, None, "ivy", "ivy", "xml", None) + ).flatMap { s => + if (s.endsWith("/")) + s.right + else + s"Don't know how to list revisions of $metadataPattern".left + } + + def fromWebPage(s: String) = { + val subDirs = coursier.core.compatibility.listWebPageSubDirectories(s) + val versions = subDirs.map(Parse.version).collect { case Some(v) => v } + val versionsInItv = versions.filter(itv.contains) + + if (versionsInItv.isEmpty) + EitherT(F.point(s"No version found for $version".left[(Artifact.Source, Project)])) + else { + val version0 = versionsInItv.max + findNoInverval(module, version0.repr, fetch) + } + } + + def artifactFor(url: String) = + Artifact( + url, + Map.empty, + Map.empty, + Attributes("", ""), + changing = true, + authentication + ) + + for { + url <- EitherT(F.point(listingUrl)) + s <- fetch(artifactFor(url)) + res <- fromWebPage(s) + } yield res + } + } + } + + def findNoInverval[F[_]]( + module: Module, + version: String, + fetch: Fetch.Content[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, (Artifact.Source, Project)] = { + val eitherArtifact: String \/ Artifact = for { url <- metadataPattern0.substitute( - variables(module, version, "ivy", "ivy", "xml", None) + variables(module, Some(version), "ivy", "ivy", "xml", None) ) } yield { var artifact = Artifact( @@ -176,7 +250,9 @@ case class IvyRepository( else proj0 - (source, proj) + source -> proj.copy( + actualVersionOpt = Some(version) + ) } } diff --git a/tests/jvm/src/test/scala/coursier/test/IvyTests.scala b/tests/jvm/src/test/scala/coursier/test/IvyTests.scala index 0b6bc6a56..ab50e3694 100644 --- a/tests/jvm/src/test/scala/coursier/test/IvyTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/IvyTests.scala @@ -9,6 +9,13 @@ object IvyTests extends TestSuite { // only tested on the JVM for lack of support of XML attributes in the platform-dependent XML stubs + val sbtRepo = IvyRepository( + "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/" + + "[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)" + + "[revision]/[type]s/[artifact](-[classifier]).[ext]", + dropInfoAttributes = true + ) + val tests = TestSuite { 'dropInfoAttributes - { CentralTests.resolutionCheck( @@ -16,17 +23,31 @@ object IvyTests extends TestSuite { "org.scala-js", "sbt-scalajs", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10") ), version = "0.6.6", - extraRepo = Some( - IvyRepository( - "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/" + - "[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)" + - "[revision]/[type]s/[artifact](-[classifier]).[ext]", - dropInfoAttributes = true - ) - ), + extraRepo = Some(sbtRepo), configuration = "default(compile)" ) } + + 'versionIntervals - { + // will likely break if new 0.6.x versions are published :-) + + val mod = Module( + "com.github.ddispaltro", "sbt-reactjs", Map("sbtVersion" -> "0.13", "scalaVersion" -> "2.10") + ) + val ver = "0.6.+" + + val expectedArtifactUrl = "https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/com.github.ddispaltro/sbt-reactjs/scala_2.10/sbt_0.13/0.6.8/jars/sbt-reactjs.jar" + + * - CentralTests.resolutionCheck( + module = mod, + version = ver, + extraRepo = Some(sbtRepo) + ) + + * - CentralTests.withArtifact(mod, ver, extraRepo = Some(sbtRepo)) { artifact => + assert(artifact.url == expectedArtifactUrl) + } + } } } diff --git a/tests/shared/src/test/resources/resolutions/com.github.ddispaltro/sbt-reactjs/sbtVersion_0.13_scalaVersion_2.10/0.6.+ b/tests/shared/src/test/resources/resolutions/com.github.ddispaltro/sbt-reactjs/sbtVersion_0.13_scalaVersion_2.10/0.6.+ new file mode 100644 index 000000000..42ad9aa4c --- /dev/null +++ b/tests/shared/src/test/resources/resolutions/com.github.ddispaltro/sbt-reactjs/sbtVersion_0.13_scalaVersion_2.10/0.6.+ @@ -0,0 +1,73 @@ +com.fasterxml.jackson.core:jackson-annotations:2.3.0:default +com.fasterxml.jackson.core:jackson-core:2.3.3:default +com.fasterxml.jackson.core:jackson-databind:2.3.3:default +com.github.ddispaltro:sbt-reactjs;sbtVersion=0.13;scalaVersion=2.10:0.6.8:compile +com.google.code.findbugs:jsr305:1.3.9:default +com.google.guava:guava:12.0:default +com.google.protobuf:protobuf-java:2.5.0:default +com.typesafe:config:1.2.1:default +com.typesafe:jse_2.10:1.1.2:default +com.typesafe:npm_2.10:1.1.1:default +com.typesafe.akka:akka-actor_2.10:2.3.11:default +com.typesafe.akka:akka-cluster_2.10:2.3.11:default +com.typesafe.akka:akka-contrib_2.10:2.3.11:default +com.typesafe.akka:akka-persistence-experimental_2.10:2.3.11:default +com.typesafe.akka:akka-remote_2.10:2.3.11:default +com.typesafe.sbt:sbt-js-engine;sbtVersion=0.13;scalaVersion=2.10:1.1.3:compile +com.typesafe.sbt:sbt-web;sbtVersion=0.13;scalaVersion=2.10:1.2.1:compile +io.apigee:rhino:1.7R5pre4:default +io.apigee.trireme:trireme-core:0.8.5:default +io.apigee.trireme:trireme-node10src:0.8.5:default +io.netty:netty:3.8.0.Final:default +io.spray:spray-json_2.10:1.3.2:default +org.apache.commons:commons-compress:1.9:default +org.apache.commons:commons-lang3:3.1:default +org.fusesource.hawtjni:hawtjni-runtime:1.8:default +org.fusesource.leveldbjni:leveldbjni:1.7:default +org.fusesource.leveldbjni:leveldbjni-all:1.7:default +org.fusesource.leveldbjni:leveldbjni-linux32:1.5:default +org.fusesource.leveldbjni:leveldbjni-linux64:1.5:default +org.fusesource.leveldbjni:leveldbjni-osx:1.5:default +org.fusesource.leveldbjni:leveldbjni-win32:1.5:default +org.fusesource.leveldbjni:leveldbjni-win64:1.5:default +org.iq80.leveldb:leveldb:0.5:default +org.iq80.leveldb:leveldb-api:0.5:default +org.scala-lang:scala-library:2.10.5:default +org.slf4j:slf4j-api:1.7.12:default +org.slf4j:slf4j-simple:1.7.12:default +org.uncommons.maths:uncommons-maths:1.2.2a:default +org.webjars:amdefine:0.1.0-2:default +org.webjars:base62js:1.0.0:default +org.webjars:esprima:13001.1.0-dev-harmony-fb:default +org.webjars:jstransform:10.1.0:default +org.webjars:mkdirp:0.5.0:default +org.webjars:npm:2.11.2:default +org.webjars:react:0.14.8:default +org.webjars:source-map:0.1.40-1:default +org.webjars:webjars-locator:0.25:default +org.webjars:webjars-locator-core:0.25:default +org.webjars.npm:amdefine:0.1.0:default +org.webjars.npm:ast-types:0.8.5:default +org.webjars.npm:base62:0.1.1:default +org.webjars.npm:commander:2.5.0:default +org.webjars.npm:commoner:0.10.3:default +org.webjars.npm:esprima-fb:15001.1.0-dev-harmony-fb:default +org.webjars.npm:glob:4.2.1:default +org.webjars.npm:graceful-fs:3.0.7:default +org.webjars.npm:iconv-lite:0.4.9:default +org.webjars.npm:inflight:1.0.4:default +org.webjars.npm:inherits:2.0.1:default +org.webjars.npm:install:0.1.8:default +org.webjars.npm:jstransform:10.1.0:default +org.webjars.npm:lru-cache:2.7.0:default +org.webjars.npm:minimatch:1.0.0:default +org.webjars.npm:minimist:0.0.8:default +org.webjars.npm:mkdirp:0.5.1:default +org.webjars.npm:once:1.3.3:default +org.webjars.npm:private:0.1.6:default +org.webjars.npm:q:1.1.2:default +org.webjars.npm:react-tools:0.13.3:default +org.webjars.npm:recast:0.10.24:default +org.webjars.npm:sigmund:1.0.1:default +org.webjars.npm:source-map:0.4.4:default +org.webjars.npm:wrappy:1.0.1:default diff --git a/tests/shared/src/test/resources/resolutions/io.get-coursier/coursier_2.11/1.0.0-SNAPSHOT b/tests/shared/src/test/resources/resolutions/io.get-coursier/coursier_2.11/1.0.0-SNAPSHOT index 5afdb21dc..160369f81 100644 --- a/tests/shared/src/test/resources/resolutions/io.get-coursier/coursier_2.11/1.0.0-SNAPSHOT +++ b/tests/shared/src/test/resources/resolutions/io.get-coursier/coursier_2.11/1.0.0-SNAPSHOT @@ -1,4 +1,5 @@ io.get-coursier:coursier_2.11:1.0.0-SNAPSHOT:compile +org.jsoup:jsoup:1.9.2:default org.scala-lang:scala-library:2.11.8:default org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4:default org.scala-lang.modules:scala-xml_2.11:1.0.4:default diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index eb75226e4..2edc9a7bd 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -89,9 +89,15 @@ object CentralTests extends TestSuite { assert(result == expected) } - def withArtifact[T](module: Module, version: String)(f: Artifact => T): Future[T] = async { + def withArtifact[T]( + module: Module, + version: String, + extraRepo: Option[Repository] = None + )( + f: Artifact => T + ): Future[T] = async { val dep = Dependency(module, version, transitive = false) - val res = await(resolve(Set(dep))) + val res = await(resolve(Set(dep), extraRepo = extraRepo)) res.artifacts match { case Seq(artifact) =>