From 1380d7a74109f976717d03c8fe140fad36534782 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 9 Nov 2016 09:28:31 -0800 Subject: [PATCH] Quick & dirty scaladex lookup --- build.sbt | 5 +- .../main/scala-2.11/coursier/cli/Helper.scala | 56 +++++- .../coursier/cli/scaladex/Scaladex.scala | 173 ++++++++++++++++++ project/Deps.scala | 1 + 4 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala diff --git a/build.sbt b/build.sbt index 677b3925b..8aece00de 100644 --- a/build.sbt +++ b/build.sbt @@ -91,7 +91,10 @@ lazy val cli = project coursierPrefix, libs ++= { if (scalaBinaryVersion.value == "2.11") - Seq(Deps.caseApp) + Seq( + Deps.caseApp, + Deps.argonautShapeless + ) else Seq() }, diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 341af33fb..3f0fa6a04 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -6,6 +6,7 @@ import java.net.{ URL, URLClassLoader } import java.util.jar.{ Manifest => JManifest } import java.util.concurrent.Executors +import coursier.cli.scaladex.Scaladex import coursier.cli.typelevel.Typelevel import coursier.ivy.IvyRepository import coursier.util.{Print, Parse} @@ -162,11 +163,60 @@ class Helper( } + val (scaladexRawDependencies, otherRawDependencies) = + rawDependencies.partition(s => s.contains("/") || !s.contains(":")) + + val scaladexModuleVersionConfigs = { + val res = scaladexRawDependencies.map { s => + val deps = Scaladex.dependencies( + s, + "2.11", + if (verbosityLevel >= 0) Console.err.println(_) else _ => () + ) + + deps.map { modVers => + val m = modVers.groupBy(_._2) + if (m.size > 1) { + val (keptVer, modVers0) = m.map { + case (v, l) => + val ver = coursier.core.Parse.version(v) + .getOrElse(???) // FIXME + + ver -> l + } + .maxBy(_._1) + + if (verbosityLevel >= 0) + Console.err.println(s"Keeping version ${keptVer.repr}") + + modVers0 + } else + modVers + } + } + + val errors = res.collect { case -\/(err) => err } + + prematureExitIf(errors.nonEmpty) { + s"Error getting scaladex infos:\n" + errors.map(" " + _).mkString("\n") + } + + res + .collect { case \/-(l) => l } + .flatten + .map { case (mod, ver) => (mod, ver, None) } + } + + val (modVerCfgErrors, moduleVersionConfigs) = - Parse.moduleVersionConfigs(rawDependencies, scalaVersion) + Parse.moduleVersionConfigs(otherRawDependencies, scalaVersion) val (intransitiveModVerCfgErrors, intransitiveModuleVersionConfigs) = Parse.moduleVersionConfigs(intransitive, scalaVersion) + def allModuleVersionConfigs = + // FIXME Order of the dependencies is not respected here (scaladex ones go first) + scaladexModuleVersionConfigs ++ moduleVersionConfigs + prematureExitIf(modVerCfgErrors.nonEmpty) { s"Cannot parse dependencies:\n" + modVerCfgErrors.map(" "+_).mkString("\n") } @@ -244,7 +294,7 @@ class Helper( (mod.organization, mod.name) }.toSet - val baseDependencies = moduleVersionConfigs.map { + val baseDependencies = allModuleVersionConfigs.map { case (module, version, configOpt) => Dependency( module, @@ -692,7 +742,7 @@ class Helper( } else { // Trying to get the main class of the first artifact val mainClassOpt = for { - (module, _, _) <- moduleVersionConfigs.headOption + (module, _, _) <- allModuleVersionConfigs.headOption mainClass <- mainClasses.collectFirst { case ((org, name), mainClass) if org == module.organization && ( diff --git a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala new file mode 100644 index 000000000..da954dc33 --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala @@ -0,0 +1,173 @@ +package coursier.cli.scaladex + +import java.net.HttpURLConnection +import java.nio.charset.StandardCharsets + +import argonaut._ +import Argonaut._ +import ArgonautShapeless._ +import coursier.Module + +import scalaz.{-\/, \/, \/-} +import scalaz.Scalaz.ToEitherOps +import scalaz.Scalaz.ToEitherOpsFromEither + +object Scaladex { + + // quick & dirty API for querying scaladex + + case class SearchResult( + /** GitHub organization */ + organization: String, + /** GitHub repository */ + repository: String, + /** Scaladex artifact names */ + artifacts: List[String] = Nil + ) + + def search(name: String, target: String, scalaVersion: String): String \/ Seq[SearchResult] = { + + val url = new java.net.URL( + // FIXME Escaping + s"https://index.scala-lang.org/api/scastie/search?q=$name&target=$target&scalaVersion=$scalaVersion" + ) + + var conn: HttpURLConnection = null + + val b = try { + conn = url.openConnection().asInstanceOf[HttpURLConnection] + // FIXME See below + // conn.setRequestProperty("Accept", "application/json") + + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } + + val s = new String(b, StandardCharsets.UTF_8) + + s.decodeEither[List[SearchResult]].disjunction + } + + case class ArtifactInfos( + /** Dependency group ID (aka organization) */ + groupId: String, + /** Dependency artifact ID (aka name or module name) */ + artifactId: String, + /** Dependency version */ + version: String + ) + + /** + * + * @param organization: GitHub organization + * @param repository: GitHub repository name + * @param artifactName: Scaladex artifact name + * @return + */ + def artifactInfos(organization: String, repository: String, artifactName: String): String \/ ArtifactInfos = { + + val url = new java.net.URL( + // FIXME Escaping + s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository&artifact=$artifactName" + ) + + var conn: HttpURLConnection = null + + val b = try { + conn = url.openConnection().asInstanceOf[HttpURLConnection] + // FIXME See below + // conn.setRequestProperty("Accept", "application/json") + + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } + + val s = new String(b, StandardCharsets.UTF_8) + + s.decodeEither[ArtifactInfos].disjunction + } + + /** + * + * @param organization: GitHub organization + * @param repository: GitHub repository name + * @return + */ + def artifactNames(organization: String, repository: String): String \/ Seq[String] = { + + val url = new java.net.URL( + // FIXME Escaping + s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository" + ) + + var conn: HttpURLConnection = null + + val b = try { + conn = url.openConnection().asInstanceOf[HttpURLConnection] + // FIXME report to scaladex, it should accept that (it currently returns JSON as text/plain) + // conn.setRequestProperty("Accept", "application/json") + + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } + + val s = new String(b, StandardCharsets.UTF_8) + + case class Result(artifacts: List[String]) + + s.decodeEither[Result].disjunction.map(_.artifacts) + } + + + /** + * Modules / versions known to the Scaladex + * + * Latest version only. + */ + def dependencies(name: String, scalaVersion: String, logger: String => Unit): String \/ Seq[(Module, String)] = { + + val idx = name.indexOf('/') + val orgNameOrError = + if (idx >= 0) { + val org = name.take(idx) + val repo = name.drop(idx + 1) + + artifactNames(org, repo).map((org, repo, _)) + } else + search(name, "JVM", scalaVersion) // FIXME Don't hardcode + .flatMap { + case Seq(first, _*) => + logger(s"Using ${first.organization}/${first.repository} for $name") + (first.organization, first.repository, first.artifacts).right + case Seq() => + s"No project found for $name".left + } + + orgNameOrError.flatMap { + case (ghOrg, ghRepo, artifactNames) => + + val moduleVersions = artifactNames.flatMap { artifactName => + artifactInfos(ghOrg, ghRepo, artifactName) match { + case -\/(err) => + logger(s"Cannot get infos about artifact $artifactName from $ghOrg/$ghRepo: $err, ignoring it") + Nil + case \/-(infos) => + logger(s"Found module ${infos.groupId}:${infos.artifactId}:${infos.version}") + Seq(Module(infos.groupId, infos.artifactId) -> infos.version) + } + } + + if (moduleVersions.isEmpty) + s"No module found for $ghOrg/$ghRepo".left + else + moduleVersions.right + } + } + +} diff --git a/project/Deps.scala b/project/Deps.scala index 2257162c7..4f0461186 100644 --- a/project/Deps.scala +++ b/project/Deps.scala @@ -16,6 +16,7 @@ object Deps { def okhttpUrlConnection = "com.squareup.okhttp" % "okhttp-urlconnection" % "2.7.5" def sbtLauncherInterface = "org.scala-sbt" % "launcher-interface" % "1.0.0" def typesafeConfig = "com.typesafe" % "config" % "1.3.1" + def argonautShapeless = "com.github.alexarchambault" %% "argonaut-shapeless_6.2" % "1.2.0-M4" def scalaAsync = Def.setting {