Merge pull request #541 from coursier/develop

Latest developments

appveyor.yml

@@ -24,8 +24,8 @@ install:
 build_script:
   - sbt ++2.11.11 clean compile coreJVM/publishLocal http-server/publishLocal
   - sbt ++2.10.6 clean compile
-  - sbt ++2.12.1 coreJVM/publishLocal cache/publishLocal # to make the scripted sbt 1.0 tests happy
-  - sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal # to make the scripted sbt 0.13 tests happy
+  - sbt ++2.12.1 coreJVM/publishLocal cache/publishLocal extra/publishLocal # to make the scripted sbt 1.0 tests happy
+  - sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal extra/publishLocal # to make the scripted sbt 0.13 tests happy
 test_script:
   - ps: Start-Job -filepath .\scripts\start-it-auth-server.ps1 -ArgumentList $pwd
   - ps: Start-Sleep -s 15 # wait for the first server to have downloaded its dependencies

build.sbt

@@ -88,8 +88,15 @@ lazy val bootstrap = project
     renameMainJar("bootstrap.jar")
   )
 
+lazy val extra = project
+  .dependsOn(coreJvm)
+  .settings(
+    shared,
+    coursierPrefix
+  )
+
 lazy val cli = project
-  .dependsOn(coreJvm, cache)
+  .dependsOn(coreJvm, cache, extra)
   .settings(
     shared,
     dontPublishIn("2.10", "2.12"),
@@ -166,7 +173,7 @@ lazy val doc = project
 
 // Don't try to compile that if you're not in 2.10
 lazy val `sbt-coursier` = project
-  .dependsOn(coreJvm, cache)
+  .dependsOn(coreJvm, cache, extra)
   .settings(plugin)
 
 // Don't try to compile that if you're not in 2.10
@@ -224,6 +231,7 @@ lazy val jvm = project
     testsJvm,
     cache,
     bootstrap,
+    extra,
     cli,
     `sbt-coursier`,
     `sbt-shading`,
@@ -262,6 +270,7 @@ lazy val coursier = project
     testsJs,
     cache,
     bootstrap,
+    extra,
     cli,
     `sbt-coursier`,
     `sbt-shading`,

cache/src/main/scala/coursier/Cache.scala

@@ -957,8 +957,6 @@ object Cache {
     }
   }
 
-  private val utf8Bom = "\ufeff"
-
   def fetch(
     cache: File = default,
     cachePolicy: CachePolicy = CachePolicy.UpdateChanging,
@@ -981,7 +979,7 @@ object Cache {
         def notFound(f: File) = Left(s"${f.getCanonicalPath} not found")
 
         def read(f: File) =
-          try Right(new String(FileUtil.readAllBytes(f), "UTF-8").stripPrefix(utf8Bom))
+          try Right(new String(FileUtil.readAllBytes(f), "UTF-8"))
           catch {
             case NonFatal(e) =>
               Left(s"Could not read (file:${f.getCanonicalPath}): ${e.getMessage}")

cli/src/main/scala-2.11/coursier/cli/Fetch.scala

@@ -23,6 +23,9 @@ final case class Fetch(
     )
   )
 
+  // Some progress lines seem to be scraped without this.
+  Console.out.flush()
+
   val out =
     if (options.classpath)
       files0

cli/src/main/scala-2.11/coursier/cli/Helper.scala

@@ -7,7 +7,7 @@ import java.util.jar.{ Manifest => JManifest }
 import java.util.concurrent.Executors
 
 import coursier.cli.scaladex.Scaladex
-import coursier.cli.typelevel.Typelevel
+import coursier.extra.Typelevel
 import coursier.ivy.IvyRepository
 import coursier.util.{Print, Parse}
 

core/jvm/src/main/scala/coursier/core/compatibility/Entities.scala

@@ -0,0 +1,99 @@
+package coursier.core.compatibility
+
+object Entities {
+
+  // Generated via https://gist.github.com/alexarchambault/79388ff31ec8cbddf6607b55ab2f6527
+
+  val entities = Vector(
+    (" ", " "),
+    ("¡", "¡"),
+    ("¢", "¢"),
+    ("£", "£"),
+    ("¤", "¤"),
+    ("¥", "¥"),
+    ("¦", "¦"),
+    ("§", "§"),
+    ("¨", "¨"),
+    ("©", "©"),
+    ("ª", "ª"),
+    ("«", "«"),
+    ("¬", "¬"),
+    ("­", "­"),
+    ("®", "®"),
+    ("¯", "¯"),
+    ("°", "°"),
+    ("±", "±"),
+    ("´", "´"),
+    ("µ", "µ"),
+    ("¶", "¶"),
+    ("·", "·"),
+    ("¸", "¸"),
+    ("º", "º"),
+    ("»", "»"),
+    ("¿", "¿"),
+    ("À", "À"),
+    ("Á", "Á"),
+    ("Â", "Â"),
+    ("Ã", "Ã"),
+    ("Ä", "Ä"),
+    ("Å", "Å"),
+    ("Æ", "Æ"),
+    ("Ç", "Ç"),
+    ("È", "È"),
+    ("É", "É"),
+    ("Ê", "Ê"),
+    ("Ë", "Ë"),
+    ("Ì", "Ì"),
+    ("Í", "Í"),
+    ("Î", "Î"),
+    ("Ï", "Ï"),
+    ("Ð", "Ð"),
+    ("Ñ", "Ñ"),
+    ("Ò", "Ò"),
+    ("Ó", "Ó"),
+    ("Ô", "Ô"),
+    ("Õ", "Õ"),
+    ("Ö", "Ö"),
+    ("×", "×"),
+    ("Ø", "Ø"),
+    ("Ù", "Ù"),
+    ("Ú", "Ú"),
+    ("Û", "Û"),
+    ("Ü", "Ü"),
+    ("Ý", "Ý"),
+    ("Þ", "Þ"),
+    ("ß", "ß"),
+    ("à", "à"),
+    ("á", "á"),
+    ("â", "â"),
+    ("ã", "ã"),
+    ("ä", "ä"),
+    ("å", "å"),
+    ("æ", "æ"),
+    ("ç", "ç"),
+    ("è", "è"),
+    ("é", "é"),
+    ("ê", "ê"),
+    ("ë", "ë"),
+    ("ì", "ì"),
+    ("í", "í"),
+    ("î", "î"),
+    ("ï", "ï"),
+    ("ð", "ð"),
+    ("ñ", "ñ"),
+    ("ò", "ò"),
+    ("ó", "ó"),
+    ("ô", "ô"),
+    ("õ", "õ"),
+    ("ö", "ö"),
+    ("÷", "÷"),
+    ("ø", "ø"),
+    ("ù", "ù"),
+    ("ú", "ú"),
+    ("û", "û"),
+    ("ü", "ü"),
+    ("ý", "ý"),
+    ("þ", "þ"),
+    ("ÿ", "ÿ")
+  )
+}

core/jvm/src/main/scala/coursier/core/compatibility/package.scala

@@ -2,6 +2,8 @@ package coursier.core
 
 import coursier.util.Xml
 
+import java.util.regex.Pattern.quote
+
 import scala.collection.JavaConverters._
 import scala.xml.{ Attribute, MetaData, Null }
 
@@ -14,9 +16,23 @@ package object compatibility {
     def letter = c.isLetter
   }
 
+  private val entityPattern = (quote("&") + "[a-zA-Z]+" + quote(";")).r
+
+  private val utf8Bom = "\ufeff"
+
   def xmlParse(s: String): Either[String, Xml.Node] = {
+
+    val content =
+      if (entityPattern.findFirstIn(s).isEmpty)
+        s
+      else
+        Entities.entities.foldLeft(s) {
+          case (s0, (target, replacement)) =>
+            s0.replace(target, replacement)
+        }
+
     def parse =
-      try Right(scala.xml.XML.loadString(s))
+      try Right(scala.xml.XML.loadString(content.stripPrefix(utf8Bom)))
       catch { case e: Exception => Left(e.toString + Option(e.getMessage).fold("")(" (" + _ + ")")) }
 
     def fromNode(node: scala.xml.Node): Xml.Node =

core/shared/src/main/scala/coursier/core/Resolution.scala

@@ -31,11 +31,7 @@ object Resolution {
 
     def fromActivation = profile.activation.isActive(properties, osInfo, jdkVersion)
 
-    val res = fromUserOrDefault.getOrElse(fromActivation)
-
-    // println(s"Profile\n$profile\n$res\n")
-
-    res
+    fromUserOrDefault.getOrElse(fromActivation)
   }
 
   /**
@@ -785,31 +781,42 @@ final case class Resolution(
     project: Project
   ): Set[ModuleVersion] = {
 
-    val approxProperties0 =
-      project.parent
-        .flatMap(projectCache.get)
-        .map(_._2.properties)
-        .fold(project.properties)(project.properties ++ _)
+    val needsParent =
+      project.parent.exists { par =>
+        val parentFound = projectCache.contains(par) || errorCache.contains(par)
+        !parentFound
+      }
 
-    val approxProperties = propertiesMap(approxProperties0) ++ projectProperties(project)
+    if (needsParent)
+      project.parent.toSet
+    else {
 
-    val profileDependencies =
-      profiles(
-        project,
-        approxProperties,
-        osInfo,
-        jdkVersion,
-        userActivations
-      ).flatMap(p => p.dependencies ++ p.dependencyManagement)
+      val approxProperties0 =
+        project.parent
+          .flatMap(projectCache.get)
+          .map(_._2.properties)
+          .fold(project.properties)(project.properties ++ _)
 
-    val modules = withProperties(
-      project.dependencies ++ project.dependencyManagement ++ profileDependencies,
-      approxProperties
-    ).collect {
-      case ("import", dep) => dep.moduleVersion
+      val approxProperties = propertiesMap(approxProperties0) ++ projectProperties(project)
+
+      val profileDependencies =
+        profiles(
+          project,
+          approxProperties,
+          osInfo,
+          jdkVersion,
+          userActivations
+        ).flatMap(p => p.dependencies ++ p.dependencyManagement)
+
+      val modules = withProperties(
+        project.dependencies ++ project.dependencyManagement ++ profileDependencies,
+        approxProperties
+      ).collect {
+        case ("import", dep) => dep.moduleVersion
+      }
+
+      modules.toSet
     }
-
-    modules.toSet ++ project.parent
   }
 
   /**

core/shared/src/main/scala/coursier/maven/MavenSource.scala

@@ -95,6 +95,8 @@ final case class MavenSource(
     }
   }
 
+  private val types = Map("sha1" -> "SHA-1", "md5" -> "MD5", "asc" -> "sig")
+
   private def artifactsKnownPublications(
     dependency: Dependency,
     project: Project,
@@ -134,40 +136,40 @@ final case class MavenSource(
       }
     }
 
-    def extensionIsExtra(publications: Seq[EnrichedPublication], ext: String, tpe: String): Seq[EnrichedPublication] = {
-
-      val (withExt, other) = publications.partition(_.publication.ext.endsWith("." + ext))
-
-      var withExtMap = withExt.map(p => (p.publication.classifier, p.publication.ext.stripSuffix("." + ext)) -> p).toMap
-
-      other.map { p =>
-        val key = (p.publication.classifier, p.publication.ext)
-        withExtMap.get(key).fold(p) { sigPub =>
-          withExtMap -= key
-          p.copy(
-            extra = p.extra + (tpe -> sigPub)
-          )
-        }
-      } ++ withExtMap.values
-    }
-
     def groupedEnrichedPublications(publications: Seq[Publication]): Seq[EnrichedPublication] = {
 
-      def helperSameName(publications: Seq[Publication]): Seq[EnrichedPublication] = {
+      def helper(publications: Seq[Publication]): Seq[EnrichedPublication] = {
 
-        val publications0 = publications.map { pub =>
-          EnrichedPublication(pub, Map())
+        var publications0 = publications
+          .map { pub =>
+            pub.ext -> EnrichedPublication(pub, Map())
+          }
+          .toMap
+
+        val byLength = publications0.toVector.sortBy(-_._1.length)
+
+        for {
+          (ext, _) <- byLength
+          idx = ext.lastIndexOf('.')
+          if idx >= 0
+          subExt = ext.substring(idx + 1)
+          baseExt = ext.substring(0, idx)
+          tpe <- types.get(subExt)
+          mainPub <- publications0.get(baseExt)
+        } {
+          val pub = publications0(ext)
+          publications0 += baseExt -> mainPub.copy(
+            extra = mainPub.extra + (tpe -> pub)
+          )
+          publications0 -= ext
         }
 
-        Seq("sha1" -> "SHA-1", "md5" -> "MD5", "asc" -> "sig").foldLeft(publications0) {
-          case (pub, (ext, tpe)) =>
-            extensionIsExtra(pub, ext, tpe)
-        }
+        publications0.values.toVector
       }
 
       publications
-        .groupBy(_.name)
-        .mapValues(helperSameName)
+        .groupBy(p => (p.name, p.classifier))
+        .mapValues(helper)
         .values
         .toVector
         .flatten

extra/src/main/scala/coursier/extra/Typelevel.scala

@@ -1,4 +1,4 @@
-package coursier.cli.typelevel
+package coursier.extra
 
 import coursier.{Dependency, Module}
 

sbt-coursier/src/main/scala/coursier/Tasks.scala

@@ -5,6 +5,7 @@ import java.net.URL
 import java.util.concurrent.{ExecutorService, Executors}
 
 import coursier.core.{Authentication, Publication}
+import coursier.extra.Typelevel
 import coursier.ivy.{IvyRepository, PropertiesPattern}
 import coursier.Keys._
 import coursier.Structure._
@@ -504,6 +505,8 @@ object Tasks {
 
       val userEnabledProfiles = mavenProfiles.value
 
+      val typelevel = scalaOrganization.value == Typelevel.typelevelOrg
+
       val startRes = Resolution(
         currentProject.dependencies.map(_._2).toSet,
         filter = Some(dep => !dep.optional),
@@ -517,7 +520,8 @@ object Tasks {
           userForceVersions ++
           forcedScalaModules(so, sv) ++
           interProjectDependencies.map(_.moduleVersion),
-        projectCache = parentProjectCache
+        projectCache = parentProjectCache,
+        mapDependencies = if (typelevel) Some(Typelevel.swap(_)) else None
       )
 
       if (verbosityLevel >= 2) {

sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/build.sbt

@@ -0,0 +1,9 @@
+scalaVersion := "2.12.2-bin-typelevel-4"
+scalaOrganization := "org.typelevel"
+scalacOptions += "-Yinduction-heuristics"
+
+libraryDependencies ++= Seq(
+  "com.47deg" %% "freestyle" % "0.1.0",
+  // CrossVersion.patch not available in sbt 0.13.8
+  compilerPlugin("org.scalamacros" % "paradise_2.12.2" % "2.1.0")
+)

sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/project/plugins.sbt

@@ -0,0 +1,11 @@
+{
+  val pluginVersion = sys.props.getOrElse(
+    "plugin.version",
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin
+    )
+  )
+
+  addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
+}

sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/src/main/scala/Foo.scala

@@ -0,0 +1,5 @@
+import freestyle._
+
+@free trait Foo {
+  def bzz: FS[java.lang.String]
+}

sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/test

@@ -0,0 +1 @@
+> compile

scripts/travis.sh

@@ -75,14 +75,14 @@ is212() {
 }
 
 runSbtCoursierTests() {
-  sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal "sbt-coursier/scripted sbt-coursier/*"
+  sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal extra/publishLocal "sbt-coursier/scripted sbt-coursier/*"
   if [ "$SCALA_VERSION" = "2.10" ]; then
     sbt ++$SCALA_VERSION "sbt-coursier/scripted sbt-coursier-0.13/*"
   fi
 }
 
 runSbtShadingTests() {
-  sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal sbt-coursier/publishLocal "sbt-shading/scripted sbt-shading/*"
+  sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal extra/publishLocal sbt-coursier/publishLocal "sbt-shading/scripted sbt-shading/*"
   if [ "$SCALA_VERSION" = "2.10" ]; then
     sbt ++$SCALA_VERSION "sbt-shading/scripted sbt-shading-0.13/*"
   fi
@@ -128,7 +128,7 @@ testLauncherJava6() {
 }
 
 testSbtCoursierJava6() {
-  sbt ++${SCALA_VERSION} coreJVM/publishLocal cache/publishLocal sbt-coursier/publishLocal
+  sbt ++${SCALA_VERSION} coreJVM/publishLocal cache/publishLocal extra/publishLocal sbt-coursier/publishLocal
 
   git clone https://github.com/alexarchambault/scalacheck-shapeless.git
   cd scalacheck-shapeless

tests/shared/src/test/resources/resolutions/org.codehaus.plexus/plexus/1.0.4

@@ -0,0 +1 @@
+org.codehaus.plexus:plexus:1.0.4:compile

tests/shared/src/test/resources/resolutions/org.kie/kie-api/6.5.0.Final

@@ -0,0 +1,2 @@
+org.kie:kie-api:6.5.0.Final:compile
+org.slf4j:slf4j-api:1.7.2:compile

tests/shared/src/test/resources/resolutions/org.yaml/snakeyaml/1.17

@@ -0,0 +1 @@
+org.yaml:snakeyaml:1.17:compile

tests/shared/src/test/scala/coursier/test/CentralTests.scala

@@ -32,6 +32,14 @@ object CentralTests extends TestSuite {
     )
       .process
       .run(fetch)
+      .map { res =>
+
+        assert(res.metadataErrors.isEmpty)
+        assert(res.conflicts.isEmpty)
+        assert(res.isDone)
+
+        res
+      }
       .runF
   }
 
@@ -155,9 +163,14 @@ object CentralTests extends TestSuite {
     assert(res.conflicts.isEmpty)
     assert(res.isDone)
 
-    val artifacts = classifierOpt.fold(res.dependencyArtifacts)(c => res.dependencyClassifiersArtifacts(Seq(c))).map(_._2).filter { a =>
-      a.`type` == artifactType
-    }
+    val artifacts = classifierOpt
+      .fold(res.dependencyArtifacts)(c => res.dependencyClassifiersArtifacts(Seq(c)))
+      .map(_._2)
+      .filter {
+        if (artifactType == "*") _ => true
+        else
+          _.`type` == artifactType
+      }
 
     f(artifacts)
   }
@@ -607,6 +620,55 @@ object CentralTests extends TestSuite {
         }
       }
     }
+
+    'entities - {
+      'odash - resolutionCheck(
+        Module("org.codehaus.plexus", "plexus"),
+        "1.0.4"
+      )
+    }
+
+    'parentBeforeImports - {
+      * - resolutionCheck(
+        Module("org.kie", "kie-api"),
+        "6.5.0.Final",
+        extraRepo = Some(MavenRepository("https://repository.jboss.org/nexus/content/repositories/public"))
+      )
+    }
+
+    'signaturesOfSignatures - {
+      val mod = Module("org.yaml", "snakeyaml")
+      val ver = "1.17"
+
+      def hasSha1(a: Artifact) = a.extra.contains("SHA-1")
+      def hasMd5(a: Artifact) = a.extra.contains("MD5")
+      def hasSig(a: Artifact) = a.extra.contains("sig")
+      def sigHasSig(a: Artifact) = a.extra.get("sig").exists(hasSig)
+
+      * - resolutionCheck(mod, ver)
+
+      * - withArtifacts(mod, ver, "*") { artifacts =>
+
+        val jarOpt = artifacts.find(_.`type` == "bundle")
+        val pomOpt = artifacts.find(_.`type` == "pom")
+
+        if (artifacts.length != 2 || jarOpt.isEmpty || pomOpt.isEmpty)
+          artifacts.foreach(println)
+
+        assert(artifacts.length == 2)
+        assert(jarOpt.nonEmpty)
+        assert(pomOpt.nonEmpty)
+
+        assert(jarOpt.forall(hasSha1))
+        assert(pomOpt.forall(hasSha1))
+        assert(jarOpt.forall(hasMd5))
+        assert(pomOpt.forall(hasMd5))
+        assert(jarOpt.forall(hasSig))
+        assert(pomOpt.forall(hasSig))
+        assert(jarOpt.forall(sigHasSig))
+        assert(pomOpt.forall(sigHasSig))
+      }
+    }
   }
 
 }