From 1380d7a74109f976717d03c8fe140fad36534782 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 9 Nov 2016 09:28:31 -0800 Subject: [PATCH 1/6] Quick & dirty scaladex lookup --- build.sbt | 5 +- .../main/scala-2.11/coursier/cli/Helper.scala | 56 +++++- .../coursier/cli/scaladex/Scaladex.scala | 173 ++++++++++++++++++ project/Deps.scala | 1 + 4 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala diff --git a/build.sbt b/build.sbt index 677b3925b..8aece00de 100644 --- a/build.sbt +++ b/build.sbt @@ -91,7 +91,10 @@ lazy val cli = project coursierPrefix, libs ++= { if (scalaBinaryVersion.value == "2.11") - Seq(Deps.caseApp) + Seq( + Deps.caseApp, + Deps.argonautShapeless + ) else Seq() }, diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 341af33fb..3f0fa6a04 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -6,6 +6,7 @@ import java.net.{ URL, URLClassLoader } import java.util.jar.{ Manifest => JManifest } import java.util.concurrent.Executors +import coursier.cli.scaladex.Scaladex import coursier.cli.typelevel.Typelevel import coursier.ivy.IvyRepository import coursier.util.{Print, Parse} @@ -162,11 +163,60 @@ class Helper( } + val (scaladexRawDependencies, otherRawDependencies) = + rawDependencies.partition(s => s.contains("/") || !s.contains(":")) + + val scaladexModuleVersionConfigs = { + val res = scaladexRawDependencies.map { s => + val deps = Scaladex.dependencies( + s, + "2.11", + if (verbosityLevel >= 0) Console.err.println(_) else _ => () + ) + + deps.map { modVers => + val m = modVers.groupBy(_._2) + if (m.size > 1) { + val (keptVer, modVers0) = m.map { + case (v, l) => + val ver = coursier.core.Parse.version(v) + .getOrElse(???) // FIXME + + ver -> l + } + .maxBy(_._1) + + if (verbosityLevel >= 0) + Console.err.println(s"Keeping version ${keptVer.repr}") + + modVers0 + } else + modVers + } + } + + val errors = res.collect { case -\/(err) => err } + + prematureExitIf(errors.nonEmpty) { + s"Error getting scaladex infos:\n" + errors.map(" " + _).mkString("\n") + } + + res + .collect { case \/-(l) => l } + .flatten + .map { case (mod, ver) => (mod, ver, None) } + } + + val (modVerCfgErrors, moduleVersionConfigs) = - Parse.moduleVersionConfigs(rawDependencies, scalaVersion) + Parse.moduleVersionConfigs(otherRawDependencies, scalaVersion) val (intransitiveModVerCfgErrors, intransitiveModuleVersionConfigs) = Parse.moduleVersionConfigs(intransitive, scalaVersion) + def allModuleVersionConfigs = + // FIXME Order of the dependencies is not respected here (scaladex ones go first) + scaladexModuleVersionConfigs ++ moduleVersionConfigs + prematureExitIf(modVerCfgErrors.nonEmpty) { s"Cannot parse dependencies:\n" + modVerCfgErrors.map(" "+_).mkString("\n") } @@ -244,7 +294,7 @@ class Helper( (mod.organization, mod.name) }.toSet - val baseDependencies = moduleVersionConfigs.map { + val baseDependencies = allModuleVersionConfigs.map { case (module, version, configOpt) => Dependency( module, @@ -692,7 +742,7 @@ class Helper( } else { // Trying to get the main class of the first artifact val mainClassOpt = for { - (module, _, _) <- moduleVersionConfigs.headOption + (module, _, _) <- allModuleVersionConfigs.headOption mainClass <- mainClasses.collectFirst { case ((org, name), mainClass) if org == module.organization && ( diff --git a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala new file mode 100644 index 000000000..da954dc33 --- /dev/null +++ b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala @@ -0,0 +1,173 @@ +package coursier.cli.scaladex + +import java.net.HttpURLConnection +import java.nio.charset.StandardCharsets + +import argonaut._ +import Argonaut._ +import ArgonautShapeless._ +import coursier.Module + +import scalaz.{-\/, \/, \/-} +import scalaz.Scalaz.ToEitherOps +import scalaz.Scalaz.ToEitherOpsFromEither + +object Scaladex { + + // quick & dirty API for querying scaladex + + case class SearchResult( + /** GitHub organization */ + organization: String, + /** GitHub repository */ + repository: String, + /** Scaladex artifact names */ + artifacts: List[String] = Nil + ) + + def search(name: String, target: String, scalaVersion: String): String \/ Seq[SearchResult] = { + + val url = new java.net.URL( + // FIXME Escaping + s"https://index.scala-lang.org/api/scastie/search?q=$name&target=$target&scalaVersion=$scalaVersion" + ) + + var conn: HttpURLConnection = null + + val b = try { + conn = url.openConnection().asInstanceOf[HttpURLConnection] + // FIXME See below + // conn.setRequestProperty("Accept", "application/json") + + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } + + val s = new String(b, StandardCharsets.UTF_8) + + s.decodeEither[List[SearchResult]].disjunction + } + + case class ArtifactInfos( + /** Dependency group ID (aka organization) */ + groupId: String, + /** Dependency artifact ID (aka name or module name) */ + artifactId: String, + /** Dependency version */ + version: String + ) + + /** + * + * @param organization: GitHub organization + * @param repository: GitHub repository name + * @param artifactName: Scaladex artifact name + * @return + */ + def artifactInfos(organization: String, repository: String, artifactName: String): String \/ ArtifactInfos = { + + val url = new java.net.URL( + // FIXME Escaping + s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository&artifact=$artifactName" + ) + + var conn: HttpURLConnection = null + + val b = try { + conn = url.openConnection().asInstanceOf[HttpURLConnection] + // FIXME See below + // conn.setRequestProperty("Accept", "application/json") + + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } + + val s = new String(b, StandardCharsets.UTF_8) + + s.decodeEither[ArtifactInfos].disjunction + } + + /** + * + * @param organization: GitHub organization + * @param repository: GitHub repository name + * @return + */ + def artifactNames(organization: String, repository: String): String \/ Seq[String] = { + + val url = new java.net.URL( + // FIXME Escaping + s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository" + ) + + var conn: HttpURLConnection = null + + val b = try { + conn = url.openConnection().asInstanceOf[HttpURLConnection] + // FIXME report to scaladex, it should accept that (it currently returns JSON as text/plain) + // conn.setRequestProperty("Accept", "application/json") + + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } + + val s = new String(b, StandardCharsets.UTF_8) + + case class Result(artifacts: List[String]) + + s.decodeEither[Result].disjunction.map(_.artifacts) + } + + + /** + * Modules / versions known to the Scaladex + * + * Latest version only. + */ + def dependencies(name: String, scalaVersion: String, logger: String => Unit): String \/ Seq[(Module, String)] = { + + val idx = name.indexOf('/') + val orgNameOrError = + if (idx >= 0) { + val org = name.take(idx) + val repo = name.drop(idx + 1) + + artifactNames(org, repo).map((org, repo, _)) + } else + search(name, "JVM", scalaVersion) // FIXME Don't hardcode + .flatMap { + case Seq(first, _*) => + logger(s"Using ${first.organization}/${first.repository} for $name") + (first.organization, first.repository, first.artifacts).right + case Seq() => + s"No project found for $name".left + } + + orgNameOrError.flatMap { + case (ghOrg, ghRepo, artifactNames) => + + val moduleVersions = artifactNames.flatMap { artifactName => + artifactInfos(ghOrg, ghRepo, artifactName) match { + case -\/(err) => + logger(s"Cannot get infos about artifact $artifactName from $ghOrg/$ghRepo: $err, ignoring it") + Nil + case \/-(infos) => + logger(s"Found module ${infos.groupId}:${infos.artifactId}:${infos.version}") + Seq(Module(infos.groupId, infos.artifactId) -> infos.version) + } + } + + if (moduleVersions.isEmpty) + s"No module found for $ghOrg/$ghRepo".left + else + moduleVersions.right + } + } + +} diff --git a/project/Deps.scala b/project/Deps.scala index 2257162c7..4f0461186 100644 --- a/project/Deps.scala +++ b/project/Deps.scala @@ -16,6 +16,7 @@ object Deps { def okhttpUrlConnection = "com.squareup.okhttp" % "okhttp-urlconnection" % "2.7.5" def sbtLauncherInterface = "org.scala-sbt" % "launcher-interface" % "1.0.0" def typesafeConfig = "com.typesafe" % "config" % "1.3.1" + def argonautShapeless = "com.github.alexarchambault" %% "argonaut-shapeless_6.2" % "1.2.0-M4" def scalaAsync = Def.setting { From f29e74126d8ee77e753936cde7f9bae161b63067 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 9 Nov 2016 09:43:48 -0800 Subject: [PATCH 2/6] Cache scaladex lookups --- .../main/scala-2.11/coursier/cli/Helper.scala | 88 ++++++++----- .../coursier/cli/scaladex/Scaladex.scala | 119 ++++++++---------- 2 files changed, 109 insertions(+), 98 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 3f0fa6a04..bd9896a62 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -166,47 +166,69 @@ class Helper( val (scaladexRawDependencies, otherRawDependencies) = rawDependencies.partition(s => s.contains("/") || !s.contains(":")) - val scaladexModuleVersionConfigs = { - val res = scaladexRawDependencies.map { s => - val deps = Scaladex.dependencies( - s, - "2.11", - if (verbosityLevel >= 0) Console.err.println(_) else _ => () + val scaladexModuleVersionConfigs = + if (scaladexRawDependencies.isEmpty) + Nil + else { + val logger = + if (verbosityLevel >= 0) + Some(new TermDisplay( + new OutputStreamWriter(System.err), + fallbackMode = loggerFallbackMode + )) + else + None + + val fetchs = cachePolicies.map(p => + Cache.fetch(cache, p, checksums = Nil, logger = logger, pool = pool, ttl = ttl0) ) - deps.map { modVers => - val m = modVers.groupBy(_._2) - if (m.size > 1) { - val (keptVer, modVers0) = m.map { - case (v, l) => - val ver = coursier.core.Parse.version(v) - .getOrElse(???) // FIXME + logger.foreach(_.init()) - ver -> l - } - .maxBy(_._1) + val scaladex = Scaladex.cached(fetchs: _*) - if (verbosityLevel >= 0) - Console.err.println(s"Keeping version ${keptVer.repr}") + val res = scaladexRawDependencies.map { s => + val deps = scaladex.dependencies( + s, + "2.11", + if (verbosityLevel >= 0) Console.err.println(_) else _ => () + ) - modVers0 - } else - modVers + deps.map { modVers => + val m = modVers.groupBy(_._2) + if (m.size > 1) { + val (keptVer, modVers0) = m.map { + case (v, l) => + val ver = coursier.core.Parse.version(v) + .getOrElse(???) // FIXME + + ver -> l + } + .maxBy(_._1) + + if (verbosityLevel >= 0) + Console.err.println(s"Keeping version ${keptVer.repr}") + + modVers0 + } else + modVers + } } + + logger.foreach(_.stop()) + + val errors = res.collect { case -\/(err) => err } + + prematureExitIf(errors.nonEmpty) { + s"Error getting scaladex infos:\n" + errors.map(" " + _).mkString("\n") + } + + res + .collect { case \/-(l) => l } + .flatten + .map { case (mod, ver) => (mod, ver, None) } } - val errors = res.collect { case -\/(err) => err } - - prematureExitIf(errors.nonEmpty) { - s"Error getting scaladex infos:\n" + errors.map(" " + _).mkString("\n") - } - - res - .collect { case \/-(l) => l } - .flatten - .map { case (mod, ver) => (mod, ver, None) } - } - val (modVerCfgErrors, moduleVersionConfigs) = Parse.moduleVersionConfigs(otherRawDependencies, scalaVersion) diff --git a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala index da954dc33..33036cb05 100644 --- a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala +++ b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala @@ -3,19 +3,17 @@ package coursier.cli.scaladex import java.net.HttpURLConnection import java.nio.charset.StandardCharsets -import argonaut._ -import Argonaut._ -import ArgonautShapeless._ -import coursier.Module +import argonaut._, Argonaut._, ArgonautShapeless._ +import coursier.core.{ Artifact, Attributes } +import coursier.{ Fetch, Module } import scalaz.{-\/, \/, \/-} import scalaz.Scalaz.ToEitherOps import scalaz.Scalaz.ToEitherOpsFromEither +import scalaz.concurrent.Task object Scaladex { - // quick & dirty API for querying scaladex - case class SearchResult( /** GitHub organization */ organization: String, @@ -25,31 +23,6 @@ object Scaladex { artifacts: List[String] = Nil ) - def search(name: String, target: String, scalaVersion: String): String \/ Seq[SearchResult] = { - - val url = new java.net.URL( - // FIXME Escaping - s"https://index.scala-lang.org/api/scastie/search?q=$name&target=$target&scalaVersion=$scalaVersion" - ) - - var conn: HttpURLConnection = null - - val b = try { - conn = url.openConnection().asInstanceOf[HttpURLConnection] - // FIXME See below - // conn.setRequestProperty("Accept", "application/json") - - coursier.Platform.readFullySync(conn.getInputStream) - } finally { - if (conn != null) - conn.disconnect() - } - - val s = new String(b, StandardCharsets.UTF_8) - - s.decodeEither[List[SearchResult]].disjunction - } - case class ArtifactInfos( /** Dependency group ID (aka organization) */ groupId: String, @@ -59,6 +32,52 @@ object Scaladex { version: String ) + def apply(): Scaladex = + Scaladex { url => + var conn: HttpURLConnection = null + + val b = try { + conn = new java.net.URL(url).openConnection().asInstanceOf[HttpURLConnection] + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } + + new String(b, StandardCharsets.UTF_8) + } + + def cached(fetch: Fetch.Content[Task]*): Scaladex = + Scaladex { + url => + def get(fetch: Fetch.Content[Task]) = + fetch( + Artifact(url, Map(), Map(), Attributes("", ""), changing = true, None) + ) + + (get(fetch.head) /: fetch.tail)(_ orElse get(_)).run.unsafePerformSync match { + case -\/(err) => + throw new Exception(s"Fetching $url: $err") + case \/-(s) => s + } + } +} + +// TODO Add F[_] type param, change `fetch` type to `String => EitherT[F, String, String]`, adjust method signatures accordingly, ... +case class Scaladex(fetch: String => String) { + + // quick & dirty API for querying scaladex + + def search(name: String, target: String, scalaVersion: String): String \/ Seq[Scaladex.SearchResult] = { + + val s = fetch( + // FIXME Escaping + s"https://index.scala-lang.org/api/scastie/search?q=$name&target=$target&scalaVersion=$scalaVersion" + ) + + s.decodeEither[List[Scaladex.SearchResult]].disjunction + } + /** * * @param organization: GitHub organization @@ -66,29 +85,14 @@ object Scaladex { * @param artifactName: Scaladex artifact name * @return */ - def artifactInfos(organization: String, repository: String, artifactName: String): String \/ ArtifactInfos = { + def artifactInfos(organization: String, repository: String, artifactName: String): String \/ Scaladex.ArtifactInfos = { - val url = new java.net.URL( + val s = fetch( // FIXME Escaping s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository&artifact=$artifactName" ) - var conn: HttpURLConnection = null - - val b = try { - conn = url.openConnection().asInstanceOf[HttpURLConnection] - // FIXME See below - // conn.setRequestProperty("Accept", "application/json") - - coursier.Platform.readFullySync(conn.getInputStream) - } finally { - if (conn != null) - conn.disconnect() - } - - val s = new String(b, StandardCharsets.UTF_8) - - s.decodeEither[ArtifactInfos].disjunction + s.decodeEither[Scaladex.ArtifactInfos].disjunction } /** @@ -99,26 +103,11 @@ object Scaladex { */ def artifactNames(organization: String, repository: String): String \/ Seq[String] = { - val url = new java.net.URL( + val s = fetch( // FIXME Escaping s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository" ) - var conn: HttpURLConnection = null - - val b = try { - conn = url.openConnection().asInstanceOf[HttpURLConnection] - // FIXME report to scaladex, it should accept that (it currently returns JSON as text/plain) - // conn.setRequestProperty("Accept", "application/json") - - coursier.Platform.readFullySync(conn.getInputStream) - } finally { - if (conn != null) - conn.disconnect() - } - - val s = new String(b, StandardCharsets.UTF_8) - case class Result(artifacts: List[String]) s.decodeEither[Result].disjunction.map(_.artifacts) From e8af9e7aba76ec996d1ef7428061a377f9f363b6 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 9 Nov 2016 10:19:38 -0800 Subject: [PATCH 3/6] Parallelize scaladex lookups --- .../main/scala-2.11/coursier/cli/Helper.scala | 13 +-- .../coursier/cli/scaladex/Scaladex.scala | 82 ++++++++++--------- 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index bd9896a62..3a143a229 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -15,8 +15,9 @@ import scala.annotation.tailrec import scala.concurrent.duration.Duration import scala.util.Try -import scalaz.{Failure, Success, \/-, -\/} +import scalaz.{Failure, Nondeterminism, Success, \/-, -\/} import scalaz.concurrent.{ Task, Strategy } +import scalaz.std.list._ object Helper { def fileRepr(f: File) = f.toString @@ -187,11 +188,11 @@ class Helper( val scaladex = Scaladex.cached(fetchs: _*) - val res = scaladexRawDependencies.map { s => + val res = Nondeterminism[Task].gather(scaladexRawDependencies.map { s => val deps = scaladex.dependencies( s, "2.11", - if (verbosityLevel >= 0) Console.err.println(_) else _ => () + if (verbosityLevel >= 2) Console.err.println(_) else _ => () ) deps.map { modVers => @@ -206,14 +207,14 @@ class Helper( } .maxBy(_._1) - if (verbosityLevel >= 0) + if (verbosityLevel >= 1) Console.err.println(s"Keeping version ${keptVer.repr}") modVers0 } else modVers - } - } + }.run + }).unsafePerformSync logger.foreach(_.stop()) diff --git a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala index 33036cb05..bc2c75c22 100644 --- a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala +++ b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala @@ -2,15 +2,17 @@ package coursier.cli.scaladex import java.net.HttpURLConnection import java.nio.charset.StandardCharsets +import java.util.concurrent.ExecutorService import argonaut._, Argonaut._, ArgonautShapeless._ import coursier.core.{ Artifact, Attributes } import coursier.{ Fetch, Module } -import scalaz.{-\/, \/, \/-} +import scalaz.{ -\/, EitherT, Monad, Nondeterminism, \/, \/- } import scalaz.Scalaz.ToEitherOps import scalaz.Scalaz.ToEitherOpsFromEither import scalaz.concurrent.Task +import scalaz.std.list._ object Scaladex { @@ -32,50 +34,50 @@ object Scaladex { version: String ) - def apply(): Scaladex = - Scaladex { url => - var conn: HttpURLConnection = null + def apply(pool: ExecutorService): Scaladex[Task] = + Scaladex({ url => + EitherT(Task({ + var conn: HttpURLConnection = null - val b = try { - conn = new java.net.URL(url).openConnection().asInstanceOf[HttpURLConnection] - coursier.Platform.readFullySync(conn.getInputStream) - } finally { - if (conn != null) - conn.disconnect() - } + val b = try { + conn = new java.net.URL(url).openConnection().asInstanceOf[HttpURLConnection] + coursier.Platform.readFullySync(conn.getInputStream) + } finally { + if (conn != null) + conn.disconnect() + } - new String(b, StandardCharsets.UTF_8) - } + new String(b, StandardCharsets.UTF_8).right[String] + })(pool)) + }, Nondeterminism[Task]) - def cached(fetch: Fetch.Content[Task]*): Scaladex = - Scaladex { + def cached(fetch: Fetch.Content[Task]*): Scaladex[Task] = + Scaladex({ url => def get(fetch: Fetch.Content[Task]) = fetch( Artifact(url, Map(), Map(), Attributes("", ""), changing = true, None) ) - (get(fetch.head) /: fetch.tail)(_ orElse get(_)).run.unsafePerformSync match { - case -\/(err) => - throw new Exception(s"Fetching $url: $err") - case \/-(s) => s - } - } + (get(fetch.head) /: fetch.tail)(_ orElse get(_)) + }, Nondeterminism[Task]) } // TODO Add F[_] type param, change `fetch` type to `String => EitherT[F, String, String]`, adjust method signatures accordingly, ... -case class Scaladex(fetch: String => String) { +case class Scaladex[F[_]](fetch: String => EitherT[F, String, String], F: Nondeterminism[F]) { + + private implicit def F0 = F // quick & dirty API for querying scaladex - def search(name: String, target: String, scalaVersion: String): String \/ Seq[Scaladex.SearchResult] = { + def search(name: String, target: String, scalaVersion: String): EitherT[F, String, Seq[Scaladex.SearchResult]] = { val s = fetch( // FIXME Escaping s"https://index.scala-lang.org/api/scastie/search?q=$name&target=$target&scalaVersion=$scalaVersion" ) - s.decodeEither[List[Scaladex.SearchResult]].disjunction + s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[List[Scaladex.SearchResult]].disjunction)) } /** @@ -85,14 +87,14 @@ case class Scaladex(fetch: String => String) { * @param artifactName: Scaladex artifact name * @return */ - def artifactInfos(organization: String, repository: String, artifactName: String): String \/ Scaladex.ArtifactInfos = { + def artifactInfos(organization: String, repository: String, artifactName: String): EitherT[F, String, Scaladex.ArtifactInfos] = { val s = fetch( // FIXME Escaping s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository&artifact=$artifactName" ) - s.decodeEither[Scaladex.ArtifactInfos].disjunction + s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[Scaladex.ArtifactInfos].disjunction)) } /** @@ -101,7 +103,7 @@ case class Scaladex(fetch: String => String) { * @param repository: GitHub repository name * @return */ - def artifactNames(organization: String, repository: String): String \/ Seq[String] = { + def artifactNames(organization: String, repository: String): EitherT[F, String, Seq[String]] = { val s = fetch( // FIXME Escaping @@ -110,7 +112,7 @@ case class Scaladex(fetch: String => String) { case class Result(artifacts: List[String]) - s.decodeEither[Result].disjunction.map(_.artifacts) + s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[Result].disjunction.map(_.artifacts))) } @@ -119,7 +121,7 @@ case class Scaladex(fetch: String => String) { * * Latest version only. */ - def dependencies(name: String, scalaVersion: String, logger: String => Unit): String \/ Seq[(Module, String)] = { + def dependencies(name: String, scalaVersion: String, logger: String => Unit): EitherT[F, String, Seq[(Module, String)]] = { val idx = name.indexOf('/') val orgNameOrError = @@ -127,22 +129,22 @@ case class Scaladex(fetch: String => String) { val org = name.take(idx) val repo = name.drop(idx + 1) - artifactNames(org, repo).map((org, repo, _)) + artifactNames(org, repo).map((org, repo, _)): EitherT[F, String, (String, String, Seq[String])] } else search(name, "JVM", scalaVersion) // FIXME Don't hardcode .flatMap { case Seq(first, _*) => logger(s"Using ${first.organization}/${first.repository} for $name") - (first.organization, first.repository, first.artifacts).right + EitherT.fromDisjunction[F]((first.organization, first.repository, first.artifacts).right): EitherT[F, String, (String, String, Seq[String])] case Seq() => - s"No project found for $name".left + EitherT.fromDisjunction[F](s"No project found for $name".left): EitherT[F, String, (String, String, Seq[String])] } orgNameOrError.flatMap { case (ghOrg, ghRepo, artifactNames) => - val moduleVersions = artifactNames.flatMap { artifactName => - artifactInfos(ghOrg, ghRepo, artifactName) match { + val moduleVersions = F.map(F.gather(artifactNames.map { artifactName => + F.map(artifactInfos(ghOrg, ghRepo, artifactName).run) { case -\/(err) => logger(s"Cannot get infos about artifact $artifactName from $ghOrg/$ghRepo: $err, ignoring it") Nil @@ -150,12 +152,14 @@ case class Scaladex(fetch: String => String) { logger(s"Found module ${infos.groupId}:${infos.artifactId}:${infos.version}") Seq(Module(infos.groupId, infos.artifactId) -> infos.version) } - } + }))(_.flatten) - if (moduleVersions.isEmpty) - s"No module found for $ghOrg/$ghRepo".left - else - moduleVersions.right + EitherT(F.map(moduleVersions) { l => + if (l.isEmpty) + s"No module found for $ghOrg/$ghRepo".left + else + l.right + }) } } From c332d4f46e7303f1c23fa1dc603ee499bbadedd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillaume=20Mass=C3=A9?= Date: Sun, 13 Nov 2016 11:24:11 +0100 Subject: [PATCH 4/6] scaladex cli api change & use scalaVersion from options * Scaladex was updated to allow developers to specify cli artifact. It will only return projects and artifacts with cli enable --- cli/src/main/scala-2.11/coursier/cli/Helper.scala | 2 +- cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 3a143a229..435ea6357 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -191,7 +191,7 @@ class Helper( val res = Nondeterminism[Task].gather(scaladexRawDependencies.map { s => val deps = scaladex.dependencies( s, - "2.11", + scalaVersion, if (verbosityLevel >= 2) Console.err.println(_) else _ => () ) diff --git a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala index bc2c75c22..53e099f55 100644 --- a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala +++ b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala @@ -74,7 +74,7 @@ case class Scaladex[F[_]](fetch: String => EitherT[F, String, String], F: Nondet val s = fetch( // FIXME Escaping - s"https://index.scala-lang.org/api/scastie/search?q=$name&target=$target&scalaVersion=$scalaVersion" + s"https://index.scala-lang.org/api/search?q=$name&target=$target&scalaVersion=$scalaVersion&cli=true" ) s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[List[Scaladex.SearchResult]].disjunction)) @@ -91,7 +91,7 @@ case class Scaladex[F[_]](fetch: String => EitherT[F, String, String], F: Nondet val s = fetch( // FIXME Escaping - s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository&artifact=$artifactName" + s"https://index.scala-lang.org/api/project?organization=$organization&repository=$repository&artifact=$artifactName" ) s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[Scaladex.ArtifactInfos].disjunction)) @@ -122,7 +122,6 @@ case class Scaladex[F[_]](fetch: String => EitherT[F, String, String], F: Nondet * Latest version only. */ def dependencies(name: String, scalaVersion: String, logger: String => Unit): EitherT[F, String, Seq[(Module, String)]] = { - val idx = name.indexOf('/') val orgNameOrError = if (idx >= 0) { From 62d3ed307ab04fca73bbcc8c66ca9928a9b91051 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 18 Mar 2017 16:37:27 +0100 Subject: [PATCH 5/6] Remove cli=true from scaladex query Makes it not find lihaoyi/Ammonite in particular --- cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala index 53e099f55..618a24e2f 100644 --- a/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala +++ b/cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala @@ -74,7 +74,7 @@ case class Scaladex[F[_]](fetch: String => EitherT[F, String, String], F: Nondet val s = fetch( // FIXME Escaping - s"https://index.scala-lang.org/api/search?q=$name&target=$target&scalaVersion=$scalaVersion&cli=true" + s"https://index.scala-lang.org/api/search?q=$name&target=$target&scalaVersion=$scalaVersion" ) s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[List[Scaladex.SearchResult]].disjunction)) @@ -107,7 +107,7 @@ case class Scaladex[F[_]](fetch: String => EitherT[F, String, String], F: Nondet val s = fetch( // FIXME Escaping - s"https://index.scala-lang.org/api/scastie/project?organization=$organization&repository=$repository" + s"https://index.scala-lang.org/api/project?organization=$organization&repository=$repository" ) case class Result(artifacts: List[String]) From 9194853749651bb67302b57e1b3e2759f19236bd Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 18 Mar 2017 16:37:51 +0100 Subject: [PATCH 6/6] More loose main class detection Makes it find the main class of scalafmt --- .../main/scala-2.11/coursier/cli/Helper.scala | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 435ea6357..f59f2431d 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -763,6 +763,10 @@ class Helper( val (_, mainClass) = mainClasses.head mainClass } else { + + // TODO Move main class detection code to the coursier-extra module to come, add non regression tests for it + // In particular, check the main class for scalafmt, scalafix, ammonite, ... + // Trying to get the main class of the first artifact val mainClassOpt = for { (module, _, _) <- allModuleVersionConfigs.headOption @@ -776,7 +780,17 @@ class Helper( } } yield mainClass - mainClassOpt.getOrElse { + def sameOrgOnlyMainClassOpt = for { + (module, _, _) <- allModuleVersionConfigs.headOption + orgMainClasses = mainClasses.collect { + case ((org, name), mainClass) + if org == module.organization => + mainClass + }.toSet + if orgMainClasses.size == 1 + } yield orgMainClasses.head + + mainClassOpt.orElse(sameOrgOnlyMainClassOpt).getOrElse { Helper.errPrintln(s"Cannot find default main class. Specify one with -M or --main.") sys.exit(255) }