diff --git a/build.sbt b/build.sbt index 3cf0bd8f0..32f31df24 100644 --- a/build.sbt +++ b/build.sbt @@ -23,6 +23,9 @@ lazy val core = crossProject ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.defaultPublications"), ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.defaultPackaging"), ProblemFilters.exclude[MissingClassProblem]("coursier.maven.MavenSource$DocSourcesArtifactExtensions"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageDirectoryElements"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageSubDirectories"), + ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageFiles"), ProblemFilters.exclude[MissingTypesProblem]("coursier.core.Project$"), ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Project.apply"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.core.Project.copy$default$13"), diff --git a/core/js/src/main/scala/coursier/core/compatibility/package.scala b/core/js/src/main/scala/coursier/core/compatibility/package.scala index 3b8659b5b..18c848656 100644 --- a/core/js/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/js/src/main/scala/coursier/core/compatibility/package.scala @@ -6,6 +6,8 @@ import org.scalajs.dom.raw.NodeList import coursier.util.Xml +import scala.collection.mutable.ListBuffer + package object compatibility { def option[A](a: js.Dynamic): Option[A] = if (js.isUndefined(a)) None @@ -93,14 +95,22 @@ package object compatibility { def encodeURIComponent(s: String): String = g.encodeURIComponent(s).asInstanceOf[String] - def listWebPageSubDirectories(url: String, page: String): Seq[String] = { - // TODO - ??? - } + // FIXME Won't work in the browser + lazy val cheerio = g.require("cheerio") - def listWebPageFiles(url: String, page: String): Seq[String] = { - // TODO - ??? - } + def listWebPageRawElements(page: String): Seq[String] = { + val jquery = cheerio.load(page) + + val links = new ListBuffer[String] + + jquery("a").each({ self: js.Dynamic => + val href = jquery(self).attr("href") + if (!js.isUndefined(href)) + links += href.asInstanceOf[String] + () + }: js.ThisFunction0[js.Dynamic, Unit]) + + links.result() + } } diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index d4fbbd74c..124266ae2 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -56,25 +56,11 @@ package object compatibility { def encodeURIComponent(s: String): String = new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString - def listWebPageDirectoryElements(url: String, page: String, directories: Boolean): Seq[String] = + def listWebPageRawElements(page: String): Seq[String] = Jsoup.parse(page) .select("a") .asScala .toVector .map(_.attr("href")) - .collect { - case elem if elem.nonEmpty && elem.endsWith("/") == directories => - elem - .stripSuffix("/") - .stripPrefix(url) - .stripPrefix(":") // bintray typically prepends these - } - .filter(n => !n.contains("/") && n != "." && n != "..") - - def listWebPageSubDirectories(url: String, page: String): Seq[String] = - listWebPageDirectoryElements(url, page, directories = true) - - def listWebPageFiles(url: String, page: String): Seq[String] = - listWebPageDirectoryElements(url, page, directories = false) } diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 8956db9f2..f1bf448b2 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -2,6 +2,7 @@ package coursier.ivy import coursier.Fetch import coursier.core._ +import coursier.util.WebPage import scala.language.higherKinds @@ -149,7 +150,7 @@ final case class IvyRepository( } def fromWebPage(url: String, s: String) = { - val subDirs = coursier.core.compatibility.listWebPageSubDirectories(url, s) + val subDirs = WebPage.listDirectories(url, s) val versions = subDirs.map(Parse.version).collect { case Some(v) => v } val versionsInItv = versions.filter(itv.contains) diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 23559f5a8..5eb816d5f 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -3,6 +3,7 @@ package coursier.maven import coursier.Fetch import coursier.core._ import coursier.core.compatibility.encodeURIComponent +import coursier.util.WebPage import scala.language.higherKinds import scalaz._ @@ -289,7 +290,7 @@ final case class MavenRepository( proj <- EitherT(F.point[String \/ Project](parseRawPom(str))) } yield { - val files = coursier.core.compatibility.listWebPageFiles(listFilesUrl, rawListFilesPage) + val files = WebPage.listFiles(listFilesUrl, rawListFilesPage) val versioning = proj .snapshotVersioning diff --git a/core/shared/src/main/scala/coursier/util/WebPage.scala b/core/shared/src/main/scala/coursier/util/WebPage.scala new file mode 100644 index 000000000..ec2c6eedc --- /dev/null +++ b/core/shared/src/main/scala/coursier/util/WebPage.scala @@ -0,0 +1,22 @@ +package coursier.util + +object WebPage { + + def listElements(url: String, page: String, directories: Boolean): Seq[String] = + coursier.core.compatibility.listWebPageRawElements(page) + .collect { + case elem if elem.nonEmpty && elem.endsWith("/") == directories => + elem + .stripSuffix("/") + .stripPrefix(url) + .stripPrefix(":") // bintray typically prepends these + } + .filter(n => !n.contains("/") && n != "." && n != "..") + + def listDirectories(url: String, page: String): Seq[String] = + listElements(url, page, directories = true) + + def listFiles(url: String, page: String): Seq[String] = + listElements(url, page, directories = false) + +} \ No newline at end of file diff --git a/package.json b/package.json index e8281dcdc..262efc4b4 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "repository": "https://github.com/alexarchambault/coursier.git", "license": "Apache-2.0", "devDependencies": { + "cheerio": "0.22.0", "xmldom": "0.1.27", "xhr2": "0.1.4" }