Merge pull request #385 from coursier/topic/extra

Stuff

build.sbt

@@ -91,7 +91,10 @@ lazy val cli = project
     coursierPrefix,
     libs ++= {
       if (scalaBinaryVersion.value == "2.11")
-        Seq(Deps.caseApp)
+        Seq(
+          Deps.caseApp,
+          Deps.argonautShapeless
+        )
       else
         Seq()
     },

cli/src/main/scala-2.11/coursier/cli/Helper.scala

@@ -6,6 +6,7 @@ import java.net.{ URL, URLClassLoader }
 import java.util.jar.{ Manifest => JManifest }
 import java.util.concurrent.Executors
 
+import coursier.cli.scaladex.Scaladex
 import coursier.cli.typelevel.Typelevel
 import coursier.ivy.IvyRepository
 import coursier.util.{Print, Parse}
@@ -14,8 +15,9 @@ import scala.annotation.tailrec
 import scala.concurrent.duration.Duration
 import scala.util.Try
 
-import scalaz.{Failure, Success, \/-, -\/}
+import scalaz.{Failure, Nondeterminism, Success, \/-, -\/}
 import scalaz.concurrent.{ Task, Strategy }
+import scalaz.std.list._
 
 object Helper {
   def fileRepr(f: File) = f.toString
@@ -161,11 +163,82 @@ class Helper(
   }
 
 
+  val (scaladexRawDependencies, otherRawDependencies) =
+    rawDependencies.partition(s => s.contains("/") || !s.contains(":"))
+
+  val scaladexModuleVersionConfigs =
+    if (scaladexRawDependencies.isEmpty)
+      Nil
+    else {
+      val logger =
+        if (verbosityLevel >= 0)
+          Some(new TermDisplay(
+            new OutputStreamWriter(System.err),
+            fallbackMode = loggerFallbackMode
+          ))
+        else
+          None
+
+      val fetchs = cachePolicies.map(p =>
+        Cache.fetch(cache, p, checksums = Nil, logger = logger, pool = pool, ttl = ttl0)
+      )
+
+      logger.foreach(_.init())
+
+      val scaladex = Scaladex.cached(fetchs: _*)
+
+      val res = Nondeterminism[Task].gather(scaladexRawDependencies.map { s =>
+        val deps = scaladex.dependencies(
+          s,
+          scalaVersion,
+          if (verbosityLevel >= 2) Console.err.println(_) else _ => ()
+        )
+
+        deps.map { modVers =>
+          val m = modVers.groupBy(_._2)
+          if (m.size > 1) {
+            val (keptVer, modVers0) = m.map {
+              case (v, l) =>
+                val ver = coursier.core.Parse.version(v)
+                  .getOrElse(???) // FIXME
+
+              ver -> l
+            }
+            .maxBy(_._1)
+
+            if (verbosityLevel >= 1)
+              Console.err.println(s"Keeping version ${keptVer.repr}")
+
+            modVers0
+          } else
+            modVers
+        }.run
+      }).unsafePerformSync
+
+      logger.foreach(_.stop())
+
+      val errors = res.collect { case -\/(err) => err }
+
+      prematureExitIf(errors.nonEmpty) {
+        s"Error getting scaladex infos:\n" + errors.map("  " + _).mkString("\n")
+      }
+
+      res
+        .collect { case \/-(l) => l }
+        .flatten
+        .map { case (mod, ver) => (mod, ver, None) }
+    }
+
+
   val (modVerCfgErrors, moduleVersionConfigs) =
-    Parse.moduleVersionConfigs(rawDependencies, scalaVersion)
+    Parse.moduleVersionConfigs(otherRawDependencies, scalaVersion)
   val (intransitiveModVerCfgErrors, intransitiveModuleVersionConfigs) =
     Parse.moduleVersionConfigs(intransitive, scalaVersion)
 
+  def allModuleVersionConfigs =
+    // FIXME Order of the dependencies is not respected here (scaladex ones go first)
+    scaladexModuleVersionConfigs ++ moduleVersionConfigs
+
   prematureExitIf(modVerCfgErrors.nonEmpty) {
     s"Cannot parse dependencies:\n" + modVerCfgErrors.map("  "+_).mkString("\n")
   }
@@ -243,7 +316,7 @@ class Helper(
     (mod.organization, mod.name)
   }.toSet
 
-  val baseDependencies = moduleVersionConfigs.map {
+  val baseDependencies = allModuleVersionConfigs.map {
     case (module, version, configOpt) =>
       Dependency(
         module,
@@ -689,9 +762,13 @@ class Helper(
         val (_, mainClass) = mainClasses.head
         mainClass
       } else {
+
+        // TODO Move main class detection code to the coursier-extra module to come, add non regression tests for it
+        // In particular, check the main class for scalafmt, scalafix, ammonite, ...
+
         // Trying to get the main class of the first artifact
         val mainClassOpt = for {
-          (module, _, _) <- moduleVersionConfigs.headOption
+          (module, _, _) <- allModuleVersionConfigs.headOption
           mainClass <- mainClasses.collectFirst {
             case ((org, name), mainClass)
               if org == module.organization && (
@@ -702,7 +779,17 @@ class Helper(
           }
         } yield mainClass
 
-        mainClassOpt.getOrElse {
+        def sameOrgOnlyMainClassOpt = for {
+          (module, _, _) <- allModuleVersionConfigs.headOption
+          orgMainClasses = mainClasses.collect {
+            case ((org, name), mainClass)
+              if org == module.organization =>
+              mainClass
+          }.toSet
+          if orgMainClasses.size == 1
+        } yield orgMainClasses.head
+
+        mainClassOpt.orElse(sameOrgOnlyMainClassOpt).getOrElse {
           Helper.errPrintln(s"Cannot find default main class. Specify one with -M or --main.")
           sys.exit(255)
         }

cli/src/main/scala-2.11/coursier/cli/scaladex/Scaladex.scala

@@ -0,0 +1,165 @@
+package coursier.cli.scaladex
+
+import java.net.HttpURLConnection
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.ExecutorService
+
+import argonaut._, Argonaut._, ArgonautShapeless._
+import coursier.core.{ Artifact, Attributes }
+import coursier.{ Fetch, Module }
+
+import scalaz.{ -\/, EitherT, Monad, Nondeterminism, \/, \/- }
+import scalaz.Scalaz.ToEitherOps
+import scalaz.Scalaz.ToEitherOpsFromEither
+import scalaz.concurrent.Task
+import scalaz.std.list._
+
+object Scaladex {
+
+  case class SearchResult(
+    /** GitHub organization */
+    organization: String,
+    /** GitHub repository */
+    repository: String,
+    /** Scaladex artifact names */
+    artifacts: List[String] = Nil
+  )
+
+  case class ArtifactInfos(
+    /** Dependency group ID (aka organization) */
+    groupId: String,
+    /** Dependency artifact ID (aka name or module name) */
+    artifactId: String,
+    /** Dependency version */
+    version: String
+  )
+
+  def apply(pool: ExecutorService): Scaladex[Task] =
+    Scaladex({ url =>
+      EitherT(Task({
+        var conn: HttpURLConnection = null
+
+        val b = try {
+          conn = new java.net.URL(url).openConnection().asInstanceOf[HttpURLConnection]
+          coursier.Platform.readFullySync(conn.getInputStream)
+        } finally {
+          if (conn != null)
+            conn.disconnect()
+        }
+
+        new String(b, StandardCharsets.UTF_8).right[String]
+      })(pool))
+    }, Nondeterminism[Task])
+
+  def cached(fetch: Fetch.Content[Task]*): Scaladex[Task] =
+    Scaladex({
+      url =>
+        def get(fetch: Fetch.Content[Task]) =
+          fetch(
+            Artifact(url, Map(), Map(), Attributes("", ""), changing = true, None)
+          )
+
+        (get(fetch.head) /: fetch.tail)(_ orElse get(_))
+    }, Nondeterminism[Task])
+}
+
+// TODO Add F[_] type param, change `fetch` type to `String => EitherT[F, String, String]`, adjust method signatures accordingly, ...
+case class Scaladex[F[_]](fetch: String => EitherT[F, String, String], F: Nondeterminism[F]) {
+
+  private implicit def F0 = F
+
+  // quick & dirty API for querying scaladex
+
+  def search(name: String, target: String, scalaVersion: String): EitherT[F, String, Seq[Scaladex.SearchResult]] = {
+
+    val s = fetch(
+      // FIXME Escaping
+      s"https://index.scala-lang.org/api/search?q=$name&target=$target&scalaVersion=$scalaVersion"
+    )
+
+    s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[List[Scaladex.SearchResult]].disjunction))
+  }
+
+  /**
+    *
+    * @param organization: GitHub organization
+    * @param repository: GitHub repository name
+    * @param artifactName: Scaladex artifact name
+    * @return
+    */
+  def artifactInfos(organization: String, repository: String, artifactName: String): EitherT[F, String, Scaladex.ArtifactInfos] = {
+
+    val s = fetch(
+      // FIXME Escaping
+      s"https://index.scala-lang.org/api/project?organization=$organization&repository=$repository&artifact=$artifactName"
+    )
+
+    s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[Scaladex.ArtifactInfos].disjunction))
+  }
+
+  /**
+    *
+    * @param organization: GitHub organization
+    * @param repository: GitHub repository name
+    * @return
+    */
+  def artifactNames(organization: String, repository: String): EitherT[F, String, Seq[String]] = {
+
+    val s = fetch(
+      // FIXME Escaping
+      s"https://index.scala-lang.org/api/project?organization=$organization&repository=$repository"
+    )
+
+    case class Result(artifacts: List[String])
+
+    s.flatMap(s => EitherT.fromDisjunction[F](s.decodeEither[Result].disjunction.map(_.artifacts)))
+  }
+
+
+  /**
+    * Modules / versions known to the Scaladex
+    *
+    * Latest version only.
+    */
+  def dependencies(name: String, scalaVersion: String, logger: String => Unit): EitherT[F, String, Seq[(Module, String)]] = {
+    val idx = name.indexOf('/')
+    val orgNameOrError =
+      if (idx >= 0) {
+        val org = name.take(idx)
+        val repo = name.drop(idx + 1)
+
+        artifactNames(org, repo).map((org, repo, _)): EitherT[F, String, (String, String, Seq[String])]
+      } else
+        search(name, "JVM", scalaVersion) // FIXME Don't hardcode
+          .flatMap {
+            case Seq(first, _*) =>
+              logger(s"Using ${first.organization}/${first.repository} for $name")
+              EitherT.fromDisjunction[F]((first.organization, first.repository, first.artifacts).right): EitherT[F, String, (String, String, Seq[String])]
+            case Seq() =>
+              EitherT.fromDisjunction[F](s"No project found for $name".left): EitherT[F, String, (String, String, Seq[String])]
+          }
+
+    orgNameOrError.flatMap {
+      case (ghOrg, ghRepo, artifactNames) =>
+
+        val moduleVersions = F.map(F.gather(artifactNames.map { artifactName =>
+          F.map(artifactInfos(ghOrg, ghRepo, artifactName).run) {
+            case -\/(err) =>
+              logger(s"Cannot get infos about artifact $artifactName from $ghOrg/$ghRepo: $err, ignoring it")
+              Nil
+            case \/-(infos) =>
+              logger(s"Found module ${infos.groupId}:${infos.artifactId}:${infos.version}")
+              Seq(Module(infos.groupId, infos.artifactId) -> infos.version)
+          }
+        }))(_.flatten)
+
+        EitherT(F.map(moduleVersions) { l =>
+          if (l.isEmpty)
+            s"No module found for $ghOrg/$ghRepo".left
+          else
+            l.right
+        })
+    }
+  }
+
+}

project/Deps.scala

@@ -16,6 +16,7 @@ object Deps {
   def okhttpUrlConnection = "com.squareup.okhttp" % "okhttp-urlconnection" % "2.7.5"
   def sbtLauncherInterface = "org.scala-sbt" % "launcher-interface" % "1.0.0"
   def typesafeConfig = "com.typesafe" % "config" % "1.3.1"
+  def argonautShapeless = "com.github.alexarchambault" %% "argonaut-shapeless_6.2" % "1.2.0-M4"
 
   def scalaAsync = Def.setting {