From f57977dcd468f896203938cbddb25fc3902844e4 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 13 Jun 2017 14:47:17 +0200 Subject: [PATCH] Add support for version ranges for sbt plugins from Maven repositories These seem to lack some maven-metadata.xml files, so require specific handling --- .../coursier/maven/MavenRepository.scala | 96 ++++++++++++++----- .../coursier/test/compatibility/package.scala | 3 + .../coursier/test/compatibility/package.scala | 15 ++- tests/metadata | 2 +- .../sbtVersion_0.13_scalaVersion_2.10/1.12.+ | 9 ++ .../scala/coursier/test/CentralTests.scala | 51 ++++++---- 6 files changed, 134 insertions(+), 42 deletions(-) create mode 100644 tests/shared/src/test/resources/resolutions/org.ensime/sbt-ensime/sbtVersion_0.13_scalaVersion_2.10/1.12.+ diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 6c5e4ee88..bdc0bea74 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -76,14 +76,11 @@ final case class MavenRepository( val root0 = if (root.endsWith("/")) root else root + "/" val source = MavenSource(root0, changing, sbtAttrStub, authentication) - private def modulePath( - module: Module, - version: String - ): Seq[String] = - module.organization.split('.').toSeq ++ Seq( - dirModuleName(module, sbtAttrStub), - version - ) + private def modulePath(module: Module): Seq[String] = + module.organization.split('.').toSeq :+ dirModuleName(module, sbtAttrStub) + + private def moduleVersionPath(module: Module, version: String): Seq[String] = + modulePath(module) :+ version private def urlFor(path: Seq[String]): String = root0 + path.map(encodeURIComponent).mkString("/") @@ -94,7 +91,7 @@ final case class MavenRepository( versioningValue: Option[String] ): Artifact = { - val path = modulePath(module, version) :+ + val path = moduleVersionPath(module, version) :+ s"${module.name}-${versioningValue getOrElse version}.pom" Artifact( @@ -136,7 +133,7 @@ final case class MavenRepository( version: String ): Option[Artifact] = { - val path = modulePath(module, version) :+ "maven-metadata.xml" + val path = moduleVersionPath(module, version) :+ "maven-metadata.xml" val artifact = Artifact( @@ -153,6 +150,52 @@ final case class MavenRepository( Some(artifact) } + private def versionsFromListing[F[_]]( + module: Module, + fetch: Fetch.Content[F] + )(implicit + F: Monad[F] + ): EitherT[F, String, Versions] = { + + val listingUrl = urlFor(modulePath(module)) + "/" + + // version listing -> changing (changes as new versions are released) + val listingArtifact = artifactFor(listingUrl, changing = true) + + fetch(listingArtifact).flatMap { listing => + + val files = WebPage.listFiles(listingUrl, listing) + val rawVersions = WebPage.listDirectories(listingUrl, listing) + + val res = + if (files.contains("maven-metadata.xml")) + -\/("maven-metadata.xml found, not listing version from directory listing") + else if (rawVersions.isEmpty) + -\/(s"No versions found at $listingUrl") + else { + val parsedVersions = rawVersions.map(Version(_)) + val nonPreVersions = parsedVersions.filter(_.items.forall { + case q: Version.Qualifier => q.level >= 0 + case _ => true + }) + + if (nonPreVersions.isEmpty) + -\/(s"Found only pre-versions at $listingUrl") + else { + val latest = nonPreVersions.max + \/-(Versions( + latest.repr, + latest.repr, + nonPreVersions.map(_.repr).toList, + None + )) + } + } + + EitherT(F.point(res)) + } + } + def versions[F[_]]( module: Module, fetch: Fetch.Content[F] @@ -239,6 +282,16 @@ final case class MavenRepository( F.map(res)(_.map(proj => proj.copy(actualVersionOpt = Some(version)))) } + private def artifactFor(url: String, changing: Boolean) = + Artifact( + url, + Map.empty, + Map.empty, + Attributes("", ""), + changing = changing, + authentication + ) + def findVersioning[F[_]]( module: Module, version: String, @@ -255,16 +308,6 @@ final case class MavenRepository( proj <- Pom.project(xml, relocationAsDependency = true) } yield proj - def artifactFor(url: String) = - Artifact( - url, - Map.empty, - Map.empty, - Attributes("", ""), - changing = changing.getOrElse(version.contains("-SNAPSHOT")), - authentication - ) - def isArtifact(fileName: String, prefix: String): Option[(String, String)] = // TODO There should be a regex for that... if (fileName.startsWith(prefix)) { @@ -287,7 +330,7 @@ final case class MavenRepository( val projectArtifact0 = projectArtifact(module, version, versioningValue) - val listFilesUrl = urlFor(modulePath(module, version)) + "/" + val listFilesUrl = urlFor(moduleVersionPath(module, version)) + "/" val listFilesArtifact = Artifact( @@ -313,7 +356,7 @@ final case class MavenRepository( for { str <- fetch(requiringDirListingProjectArtifact) - rawListFilesPageOpt <- EitherT(F.map(fetch(artifactFor(listFilesUrl)).run) { + rawListFilesPageOpt <- EitherT(F.map(fetch(artifactFor(listFilesUrl, changing.getOrElse(version.contains("-SNAPSHOT")))).run) { e => \/-(e.toOption): String \/ Option[String] }) proj0 <- EitherT(F.point[String \/ Project](parseRawPom(str))) @@ -383,7 +426,14 @@ final case class MavenRepository( case None => findNoInterval(module, version, fetch).map((source, _)) case Some(itv) => - versions(module, fetch).flatMap { versions0 => + def v = versions(module, fetch) + val v0 = + if (changing.forall(!_) && module.attributes.contains("scalaVersion") && module.attributes.contains("sbtVersion")) + versionsFromListing(module, fetch).orElse(v) + else + v + + v0.flatMap { versions0 => val eitherVersion = { val release = Version(versions0.release) diff --git a/tests/js/src/test/scala/coursier/test/compatibility/package.scala b/tests/js/src/test/scala/coursier/test/compatibility/package.scala index 8c976de40..ab3b31bce 100644 --- a/tests/js/src/test/scala/coursier/test/compatibility/package.scala +++ b/tests/js/src/test/scala/coursier/test/compatibility/package.scala @@ -48,4 +48,7 @@ package object compatibility { } } } + + def tryCreate(path: String, content: String): Unit = {} + } diff --git a/tests/jvm/src/test/scala/coursier/test/compatibility/package.scala b/tests/jvm/src/test/scala/coursier/test/compatibility/package.scala index 5a97d0db6..f7179df15 100644 --- a/tests/jvm/src/test/scala/coursier/test/compatibility/package.scala +++ b/tests/jvm/src/test/scala/coursier/test/compatibility/package.scala @@ -1,6 +1,6 @@ package coursier.test -import java.io.{FileNotFoundException, InputStream} +import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import coursier.util.TestEscape @@ -71,4 +71,17 @@ package object compatibility { } } + private lazy val baseResources = { + val dir = Paths.get("tests/shared/src/test/resources") + assert(Files.isDirectory(dir)) + dir + } + + def tryCreate(path: String, content: String): Unit = + if (fillChunks) { + val path0 = baseResources.resolve(path) + Files.createDirectories(path0.getParent) + Files.write(path0, content.getBytes(StandardCharsets.UTF_8)) + } + } diff --git a/tests/metadata b/tests/metadata index f932032f9..1c0303590 160000 --- a/tests/metadata +++ b/tests/metadata @@ -1 +1 @@ -Subproject commit f932032f9cf013a6d7fe205b2a8d53229c926162 +Subproject commit 1c030359018e600bc4d33c562a4ab616b6570c63 diff --git a/tests/shared/src/test/resources/resolutions/org.ensime/sbt-ensime/sbtVersion_0.13_scalaVersion_2.10/1.12.+ b/tests/shared/src/test/resources/resolutions/org.ensime/sbt-ensime/sbtVersion_0.13_scalaVersion_2.10/1.12.+ new file mode 100644 index 000000000..50a95fd8c --- /dev/null +++ b/tests/shared/src/test/resources/resolutions/org.ensime/sbt-ensime/sbtVersion_0.13_scalaVersion_2.10/1.12.+ @@ -0,0 +1,9 @@ +org.ensime:sbt-ensime;sbtVersion=0.13;scalaVersion=2.10:1.12.12:compile +org.jsoup:jsoup:1.10.2:compile +org.scala-lang:scala-library:2.10.6:compile +org.scala-lang:scala-reflect:2.10.4:compile +org.scalamacros:quasiquotes_2.10:2.1.0:compile +org.scalariform:scalariform_2.10:0.1.4:compile +org.scalaz:scalaz-concurrent_2.10:7.2.12:compile +org.scalaz:scalaz-core_2.10:7.2.12:compile +org.scalaz:scalaz-effect_2.10:7.2.12:compile \ No newline at end of file diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index edd02a465..befd21918 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -69,27 +69,25 @@ abstract class CentralTests extends TestSuite { case (k, v) => k + "_" + v }.mkString("_") - val expected = - await( - textResource( - Seq( - "resolutions", - module.organization, - module.name, - attrPathPart, - version + ( - if (configuration.isEmpty) - "" - else - "_" + configuration.replace('(', '_').replace(')', '_') - ) - ).filter(_.nonEmpty).mkString("/") - ) - ).split('\n').toSeq + val path = Seq( + "resolutions", + module.organization, + module.name, + attrPathPart, + version + ( + if (configuration.isEmpty) + "" + else + "_" + configuration.replace('(', '_').replace(')', '_') + ) + ).filter(_.nonEmpty).mkString("/") + + def tryRead = textResource(path) val dep = Dependency(module, version, configuration = configuration) val res = await(resolve(Set(dep), extraRepo = extraRepo, profiles = profiles)) + // making that lazy makes scalac crash in 2.10 with scalajs val result = res .minDependencies .toVector @@ -109,6 +107,15 @@ abstract class CentralTests extends TestSuite { Seq(org, name, ver, cfg).mkString(":") } + val expected = + await( + tryRead.recoverWith { + case _: Exception => + tryCreate(path, result.mkString("\n")) + tryRead + } + ).split('\n').toSeq + for (((e, r), idx) <- expected.zip(result).zipWithIndex if e != r) println(s"Line ${idx + 1}:\n expected: $e\n got: $r") @@ -723,6 +730,16 @@ abstract class CentralTests extends TestSuite { } } } + + 'sbtPluginVersionRange - { + val mod = Module("org.ensime", "sbt-ensime", attributes = Map("scalaVersion" -> "2.10", "sbtVersion" -> "0.13")) + val ver = "1.12.+" + + * - { + if (isActualCentral) // doesn't work via proxies, which don't list all the upstream available versions + resolutionCheck(mod, ver) + } + } } }