Refactor web page link listing logic, implement it on the scalajs side

build.sbt

@@ -23,6 +23,9 @@ lazy val core = crossProject
         ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.defaultPublications"),
         ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.defaultPackaging"),
         ProblemFilters.exclude[MissingClassProblem]("coursier.maven.MavenSource$DocSourcesArtifactExtensions"),
+        ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageDirectoryElements"),
+        ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageSubDirectories"),
+        ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageFiles"),
         ProblemFilters.exclude[MissingTypesProblem]("coursier.core.Project$"),
         ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Project.apply"),
         ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.core.Project.copy$default$13"),

core/js/src/main/scala/coursier/core/compatibility/package.scala

@@ -6,6 +6,8 @@ import org.scalajs.dom.raw.NodeList
 
 import coursier.util.Xml
 
+import scala.collection.mutable.ListBuffer
+
 package object compatibility {
   def option[A](a: js.Dynamic): Option[A] =
     if (js.isUndefined(a)) None
@@ -93,14 +95,22 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     g.encodeURIComponent(s).asInstanceOf[String]
 
-  def listWebPageSubDirectories(url: String, page: String): Seq[String] = {
-    // TODO
-    ???
-  }
+  // FIXME Won't work in the browser
+  lazy val cheerio = g.require("cheerio")
 
-  def listWebPageFiles(url: String, page: String): Seq[String] = {
-    // TODO
-    ???
-  }
+  def listWebPageRawElements(page: String): Seq[String] = {
 
+    val jquery = cheerio.load(page)
+
+    val links = new ListBuffer[String]
+
+    jquery("a").each({ self: js.Dynamic =>
+      val href = jquery(self).attr("href")
+      if (!js.isUndefined(href))
+        links += href.asInstanceOf[String]
+      ()
+    }: js.ThisFunction0[js.Dynamic, Unit])
+
+    links.result()
+  }
 }

core/jvm/src/main/scala/coursier/core/compatibility/package.scala

@@ -56,25 +56,11 @@ package object compatibility {
   def encodeURIComponent(s: String): String =
     new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString
 
-  def listWebPageDirectoryElements(url: String, page: String, directories: Boolean): Seq[String] =
+  def listWebPageRawElements(page: String): Seq[String] =
     Jsoup.parse(page)
       .select("a")
       .asScala
       .toVector
       .map(_.attr("href"))
-      .collect {
-        case elem if elem.nonEmpty && elem.endsWith("/") == directories =>
-          elem
-            .stripSuffix("/")
-            .stripPrefix(url)
-            .stripPrefix(":") // bintray typically prepends these
-      }
-      .filter(n => !n.contains("/") && n != "." && n != "..")
-
-  def listWebPageSubDirectories(url: String, page: String): Seq[String] =
-    listWebPageDirectoryElements(url, page, directories = true)
-
-  def listWebPageFiles(url: String, page: String): Seq[String] =
-    listWebPageDirectoryElements(url, page, directories = false)
 
 }

core/shared/src/main/scala/coursier/ivy/IvyRepository.scala

@@ -2,6 +2,7 @@ package coursier.ivy
 
 import coursier.Fetch
 import coursier.core._
+import coursier.util.WebPage
 
 import scala.language.higherKinds
 
@@ -149,7 +150,7 @@ final case class IvyRepository(
             }
 
             def fromWebPage(url: String, s: String) = {
-              val subDirs = coursier.core.compatibility.listWebPageSubDirectories(url, s)
+              val subDirs = WebPage.listDirectories(url, s)
               val versions = subDirs.map(Parse.version).collect { case Some(v) => v }
               val versionsInItv = versions.filter(itv.contains)
 

core/shared/src/main/scala/coursier/maven/MavenRepository.scala

@@ -3,6 +3,7 @@ package coursier.maven
 import coursier.Fetch
 import coursier.core._
 import coursier.core.compatibility.encodeURIComponent
+import coursier.util.WebPage
 
 import scala.language.higherKinds
 import scalaz._
@@ -289,7 +290,7 @@ final case class MavenRepository(
       proj <- EitherT(F.point[String \/ Project](parseRawPom(str)))
     } yield {
 
-      val files = coursier.core.compatibility.listWebPageFiles(listFilesUrl, rawListFilesPage)
+      val files = WebPage.listFiles(listFilesUrl, rawListFilesPage)
 
       val versioning = proj
         .snapshotVersioning

core/shared/src/main/scala/coursier/util/WebPage.scala

@@ -0,0 +1,22 @@
+package coursier.util
+
+object WebPage {
+
+  def listElements(url: String, page: String, directories: Boolean): Seq[String] =
+    coursier.core.compatibility.listWebPageRawElements(page)
+      .collect {
+        case elem if elem.nonEmpty && elem.endsWith("/") == directories =>
+          elem
+            .stripSuffix("/")
+            .stripPrefix(url)
+            .stripPrefix(":") // bintray typically prepends these
+      }
+      .filter(n => !n.contains("/") && n != "." && n != "..")
+
+  def listDirectories(url: String, page: String): Seq[String] =
+    listElements(url, page, directories = true)
+
+  def listFiles(url: String, page: String): Seq[String] =
+    listElements(url, page, directories = false)
+
+}

package.json

@@ -2,6 +2,7 @@
   "repository": "https://github.com/alexarchambault/coursier.git",
   "license": "Apache-2.0",
   "devDependencies": {
+    "cheerio": "0.22.0",
     "xmldom": "0.1.27",
     "xhr2": "0.1.4"
   }