Merge pull request #326 from alexarchambault/topic/develop

Fixes

.ci/travis.sh

@@ -41,7 +41,7 @@ function isMasterOrDevelop() {
 
 # TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
 
-SBT_COMMANDS="compile test"
+SBT_COMMANDS="compile test it:test"
 
 if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then
   SBT_COMMANDS="$SBT_COMMANDS publishLocal" # to make the scripted tests happy

appveyor.yml

@@ -20,8 +20,8 @@ build_script:
   - sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal # to make the scripted tests happy
 test_script:
   - ps: Start-Job { & java -jar -noverify C:\projects\coursier\coursier launch -r http://dl.bintray.com/scalaz/releases io.get-coursier:http-server-java7_2.11:1.0.0-SNAPSHOT -- -d /C:/projects/coursier/tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm -v }
-  - sbt ++2.11.8 testsJVM/test # Would node be around for testsJS/test?
+  - sbt ++2.11.8 testsJVM/test testsJVM/it:test # Would node be around for testsJS/test?
-  - sbt ++2.10.6 testsJVM/test plugin/scripted
+  - sbt ++2.10.6 testsJVM/test testsJVM/it:test plugin/scripted
 cache:
   - C:\sbt\
   - C:\Users\appveyor\.ivy2

build.sbt

@@ -6,6 +6,8 @@ import MimaKeys.{ previousArtifacts, binaryIssueFilters }
 
 val binaryCompatibilityVersion = "1.0.0-M7"
 
+lazy val IntegrationTest = config("it") extend Test
+
 lazy val releaseSettings = Seq(
   publishMavenStyle := true,
   licenses := Seq("Apache 2.0" -> url("http://opensource.org/licenses/Apache-2.0")),
@@ -249,11 +251,13 @@ lazy val tests = crossProject
   .dependsOn(core)
   .settings(commonSettings: _*)
   .settings(noPublishSettings: _*)
+  .configs(IntegrationTest)
+  .settings(Defaults.itSettings: _*)
   .settings(
     name := "coursier-tests",
     libraryDependencies ++= Seq(
       "org.scala-lang.modules" %% "scala-async" % "0.9.5" % "provided",
-      "com.lihaoyi" %%% "utest" % "0.4.3" % "test"
+      "com.lihaoyi" %%% "utest" % "0.4.3" % "test,it"
     ),
     unmanagedResourceDirectories in Test += (baseDirectory in LocalRootProject).value / "tests" / "shared" / "src" / "test" / "resources",
     testFrameworks += new TestFramework("utest.runner.Framework")

cli/src/main/scala-2.11/coursier/cli/Options.scala

@@ -61,10 +61,10 @@ case class CommonOptions(
   @Value("configuration")
   @Short("c")
     defaultConfiguration: String = "default(compile)",
-  @Help("Default artifact type (make it empty to follow POM packaging - default: jar)")
+  @Help("Default artifact type (default: follow packaging infos, else default to jar)")
   @Value("type")
   @Short("a")
-    defaultArtifactType: String = "jar",
+    defaultArtifactType: String = "",
   @Help("Maximum number of parallel downloads (default: 6)")
   @Short("n")
     parallel: Int = 6,

core/shared/src/main/scala/coursier/maven/MavenRepository.scala

@@ -69,6 +69,10 @@ object MavenRepository {
     } else
       module.name
 
+  val ignorePackaging = Set(
+    Module("org.apache.zookeeper", "zookeeper", Map.empty)
+  )
+
 }
 
 case class MavenRepository(
@@ -76,7 +80,8 @@ case class MavenRepository(
   changing: Option[Boolean] = None,
   /** Hackish hack for sbt plugins mainly - what this does really sucks */
   sbtAttrStub: Boolean = false,
-  authentication: Option[Authentication] = None
+  authentication: Option[Authentication] = None,
+  packagingBlacklist: Set[Module] = MavenRepository.ignorePackaging
 ) extends Repository {
 
   import Repository._
@@ -260,7 +265,11 @@ case class MavenRepository(
             _ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found")
             proj <- Pom.project(xml)
           } yield {
-            val packagingOpt = Pom.packagingOpt(xml)
+            val packagingOpt =
+              if (packagingBlacklist(module))
+                None
+              else
+                Pom.packagingOpt(xml)
 
             proj.copy(
               configurations = defaultConfigurations,

plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala

@@ -59,7 +59,7 @@ object CoursierPlugin extends AutoPlugin {
   override lazy val projectSettings = Seq(
     coursierParallelDownloads := 6,
     coursierMaxIterations := 50,
-    coursierDefaultArtifactType := "jar",
+    coursierDefaultArtifactType := "",
     coursierChecksums := Seq(Some("SHA-1"), None),
     coursierArtifactsChecksums := Seq(None),
     coursierCachePolicies := CachePolicy.default,

plugin/src/main/scala-2.10/coursier/ResolutionError.scala

@@ -96,16 +96,16 @@ object ResolutionError {
         .toVector
         .sortBy(_._1)
 
-      val b = new StringBuilder
+      val b = new ArrayBuffer[String]
 
       for ((type0, errors) <- groupedArtifactErrors) {
-        b ++= s"${errors.size} $type0"
+        b += s"${errors.size} $type0"
         if (verbose)
           for (err <- errors)
-            b ++= "  " + err
+            b += "  " + err
       }
 
-      b.result()
+      b.mkString("\n")
     }
   }
 }

plugin/src/main/scala-2.10/coursier/SbtScalaJarsRepository.scala

@@ -0,0 +1,114 @@
+package coursier
+
+import java.io.File
+
+import coursier.core.Publication
+
+import scalaz.{ EitherT, Monad }
+import scalaz.Scalaz.ToEitherOps
+
+object SbtScalaJarsRepository {
+
+  // Will break in 2.11, where scala-parser-combinators_2.11 and scala-xml_2.11, with different
+  // org and versions, are thrown into the mix.
+  // To handle these well, we would need to fetch actual infos about the scala-* dependencies
+  // from actual repos, and use that from SbtScalaJarsRepository.
+
+  val looseDependencies = Map(
+    "scala-compiler" -> Set(
+      "scala-library",
+      "scala-reflect"
+    ),
+    "scala-reflect" -> Set(
+      "scala-library"
+    )
+  )
+
+}
+
+case class SbtScalaJarsRepository(
+  scalaOrg: String,
+  scalaVersion: String,
+  jars: Seq[File]
+) extends Repository { repo =>
+
+  val foundNames = jars.collect {
+    case jar if jar.getName.endsWith(".jar") =>
+      jar.getName.stripSuffix(".jar")
+  }.toSet
+
+  val dependencies = SbtScalaJarsRepository.looseDependencies
+    .filterKeys(foundNames)
+    .mapValues(_.filter(foundNames))
+
+  val artifacts = jars.collect {
+    case jar if jar.getName.endsWith(".jar") =>
+      val name = jar.getName.stripSuffix(".jar")
+      val mod = Module(scalaOrg, name)
+
+      val proj = Project(
+        mod,
+        scalaVersion,
+        dependencies.getOrElse(name, Set.empty[String]).toVector.map { depName =>
+          val dep = Dependency(Module(scalaOrg, depName), scalaVersion)
+          "compile" -> dep
+        },
+        MavenRepository.defaultConfigurations,
+        None,
+        Nil,
+        Nil,
+        Nil,
+        None,
+        None,
+        None,
+        Seq("compile" -> Publication(name, "jar", "jar", "")),
+        Info("", "", Nil, Nil, None)
+      )
+
+      (mod, scalaVersion) -> ((proj, jar))
+  }.toMap
+
+  val source: Artifact.Source = new Artifact.Source {
+    def artifacts(
+      dependency: Dependency,
+      project: Project,
+      overrideClassifiers: Option[Seq[String]]
+    ) =
+      if (overrideClassifiers.isEmpty)
+        repo.artifacts.get(project.moduleVersion) match {
+          case Some((_, f)) =>
+            Seq(
+              Artifact(
+                f.toURI.toString,
+                Map.empty,
+                Map.empty,
+                Attributes("jar", ""),
+                changing = true,
+                None
+              )
+            )
+          case None =>
+            Nil
+        }
+      else
+        Nil
+  }
+
+  def find[F[_]](
+    module: Module,
+    version: String,
+    fetch: Fetch.Content[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, (Artifact.Source, Project)] = {
+
+    val res = artifacts.get((module, version)) match {
+      case None =>
+        s"not found in internal SBT scala JARs: $module:$version".left
+      case Some((p, _)) =>
+        (source, p).right
+    }
+
+    EitherT(F.point(res))
+  }
+}

plugin/src/main/scala-2.10/coursier/Tasks.scala

@@ -6,7 +6,7 @@ import java.nio.file.Files
 import java.util.concurrent.{ ExecutorService, Executors }
 
 import coursier.core.{ Authentication, Publication }
-import coursier.ivy.IvyRepository
+import coursier.ivy.{ IvyRepository, PropertiesPattern }
 import coursier.Keys._
 import coursier.Structure._
 import coursier.maven.WritePom
@@ -19,6 +19,7 @@ import sbt.Keys._
 
 import scala.collection.mutable
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.util.Try
 
 import scalaz.{ \/-, -\/ }
@@ -258,6 +259,40 @@ object Tasks {
 
   private def createLogger() = new TermDisplay(new OutputStreamWriter(System.err))
 
+  private lazy val globalPluginPattern = {
+
+    val props = sys.props.toMap
+
+    val extraProps = new ArrayBuffer[(String, String)]
+
+    def addUriProp(key: String): Unit =
+      for (b <- props.get(key)) {
+        val uri = new File(b).toURI.toString
+        extraProps += s"$key.uri" -> uri
+      }
+
+    addUriProp("sbt.global.base")
+    addUriProp("user.home")
+
+    // FIXME get the 0.13 automatically?
+    val s = s"$${sbt.global.base.uri-$${user.home.uri}/.sbt/0.13}/plugins/target/resolution-cache/" +
+      "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]"
+
+    val p = PropertiesPattern.parse(s) match {
+      case -\/(err) =>
+        throw new Exception(s"Cannot parse pattern $s: $err")
+      case \/-(p) =>
+        p
+    }
+
+    p.substituteProperties(props ++ extraProps) match {
+      case -\/(err) =>
+        throw new Exception(err)
+      case \/-(p) =>
+        p
+    }
+  }
+
   def resolutionTask(
     sbtClassifiers: Boolean = false
   ) = Def.task {
@@ -400,9 +435,8 @@ object Tasks {
           log.info(s"  ${p.module}:${p.version}")
       }
 
-      val globalPluginsRepo = IvyRepository(
+      val globalPluginsRepo = IvyRepository.fromPattern(
-        new File(sys.props("user.home") + "/.sbt/0.13/plugins/target/resolution-cache/").toURI.toString +
+        globalPluginPattern,
-          "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]",
         withChecksums = false,
         withSignatures = false,
         withArtifacts = false
@@ -410,8 +444,30 @@ object Tasks {
 
       val interProjectRepo = InterProjectRepository(interProjectDependencies)
 
+      val internalSbtScalaProvider = appConfiguration.value.provider.scalaProvider
+      val internalSbtScalaJarsRepo = SbtScalaJarsRepository(
+        so, // this seems plain wrong - this assumes that the scala org of the project is the same
+            // as the one that started SBT. This will scrap the scala org specific JARs by the ones
+            // that booted SBT, even if the latter come from the standard org.scala-lang org.
+            // But SBT itself does it this way, and not doing so may make two different versions
+            // of the scala JARs land in the classpath...
+        internalSbtScalaProvider.version(),
+        internalSbtScalaProvider.jars()
+      )
+
+      val ivyHome = sys.props.getOrElse(
+        "ivy.home",
+        new File(sys.props("user.home")).toURI.getPath + ".ivy2"
+      )
+
+      val sbtIvyHome = sys.props.getOrElse(
+        "sbt.ivy.home",
+        ivyHome
+      )
+
       val ivyProperties = Map(
-        "ivy.home" -> (new File(sys.props("user.home")).toURI.getPath + ".ivy2")
+        "ivy.home" -> ivyHome,
+        "sbt.ivy.home" -> sbtIvyHome
       ) ++ sys.props
 
       val useSbtCredentials = coursierUseSbtCredentials.value
@@ -479,7 +535,7 @@ object Tasks {
         }
       }
 
-      val internalRepositories = Seq(globalPluginsRepo, interProjectRepo)
+      val internalRepositories = Seq(globalPluginsRepo, interProjectRepo, internalSbtScalaJarsRepo)
 
       val repositories =
         internalRepositories ++

plugin/src/sbt-test/sbt-coursier/aar-packaging/build.sbt

@@ -0,0 +1,13 @@
+scalaVersion := "2.11.8"
+
+libraryDependencies += ("com.rengwuxian.materialedittext" % "library" % "2.1.4")
+  .exclude("com.android.support", "support-v4")
+  .exclude("com.android.support", "support-annotations")
+  .exclude("com.android.support", "appcompat-v7")
+
+coursierCachePolicies := {
+  if (sys.props("os.name").startsWith("Windows"))
+    coursierCachePolicies.value
+  else
+    Seq(coursier.CachePolicy.ForceDownload)
+}

plugin/src/sbt-test/sbt-coursier/aar-packaging/project/plugins.sbt

@@ -0,0 +1,11 @@
+{
+  val pluginVersion = sys.props.getOrElse(
+    "plugin.version",
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin
+    )
+  )
+
+  addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
+}

plugin/src/sbt-test/sbt-coursier/aar-packaging/src/main/scala/Main.scala

@@ -0,0 +1,6 @@
+import java.io.File
+import java.nio.file.Files
+
+object Main extends App {
+  Files.write(new File("output").toPath, "OK".getBytes("UTF-8"))
+}

plugin/src/sbt-test/sbt-coursier/aar-packaging/test

@@ -0,0 +1,3 @@
+$ delete output
+> run
+$ exists output

plugin/src/sbt-test/sbt-coursier/scala-jars/build.sbt

@@ -0,0 +1,8 @@
+scalaVersion := appConfiguration.value.provider.scalaProvider.version
+
+coursierCachePolicies := {
+  if (sys.props("os.name").startsWith("Windows"))
+    coursierCachePolicies.value
+  else
+    Seq(coursier.CachePolicy.ForceDownload)
+}

plugin/src/sbt-test/sbt-coursier/scala-jars/project/plugins.sbt

@@ -0,0 +1,11 @@
+{
+  val pluginVersion = sys.props.getOrElse(
+    "plugin.version",
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin
+    )
+  )
+
+  addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion)
+}

plugin/src/sbt-test/sbt-coursier/scala-jars/src/main/scala/Main.scala

@@ -0,0 +1,43 @@
+import java.io.File
+import java.nio.file.Files
+
+object Main extends App {
+
+  val cp = new collection.mutable.ArrayBuffer[File]
+
+  def buildCp(loader: ClassLoader): Unit =
+    if (loader != null) {
+      loader match {
+        case u: java.net.URLClassLoader =>
+          cp ++= u.getURLs
+            .map(_.toURI)
+            .map(new File(_))
+        case _ =>
+      }
+
+      buildCp(loader.getParent)
+    }
+
+  buildCp(Thread.currentThread().getContextClassLoader)
+
+  val sbtBase = new File(sys.props.getOrElse(
+    "sbt.global.base",
+    sys.props("user.home") + "/.sbt"
+  ))
+  val prefix = new File(sbtBase, "boot").getAbsolutePath
+
+  def fromBootAndUnique(name: String): Unit = {
+    val jars = cp.filter(_.getName.startsWith(name)).distinct
+    assert(jars.length == 1, s"Found 0 or multiple JARs for $name: $jars")
+
+    val Seq(jar) = jars
+
+    assert(jar.getAbsolutePath.startsWith(prefix), s"JAR for $name ($jar) not under $prefix")
+  }
+
+  fromBootAndUnique("scala-library")
+  fromBootAndUnique("scala-reflect")
+  fromBootAndUnique("scala-compiler")
+
+  Files.write(new File("output").toPath, "OK".getBytes("UTF-8"))
+}

plugin/src/sbt-test/sbt-coursier/scala-jars/test

@@ -0,0 +1,3 @@
+$ delete output
+> run
+$ exists output

project/plugins.sbt

@@ -3,7 +3,7 @@ addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.11")
 addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.1.0")
 addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.4.2")
-addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M12")
+addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M13")
 addSbtPlugin("com.typesafe.sbt" % "sbt-proguard" % "0.2.2")
 addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.1.9")
 libraryDependencies += "org.scala-sbt" % "scripted-plugin" % sbtVersion.value

tests/jvm/src/it/scala/coursier/test/HttpAuthenticationTests.scala

@@ -0,0 +1,70 @@
+package coursier.test
+
+import utest._
+
+import coursier.core.Authentication
+import coursier.maven.MavenRepository
+
+object HttpAuthenticationTests extends TestSuite {
+
+  val tests = TestSuite {
+    'httpAuthentication - {
+      // requires an authenticated HTTP server to be running on localhost:8080 with user 'user'
+      // and password 'pass'
+
+      val address = "localhost:8080"
+      val user = "user"
+      val password = "pass"
+
+      def printErrorMessage() =
+        Console.err.println(
+          Console.RED +
+            s"HTTP authentication tests require a running HTTP server on $address, requiring " +
+            s"basic authentication with user '$user' and password '$password', serving the right " +
+            "files.\n" + Console.RESET +
+            "Run one from the coursier sources with\n" +
+            "  ./coursier launch -r http://dl.bintray.com/scalaz/releases " +
+            "io.get-coursier:simple-web-server_2.11:1.0.0-M12 -- " +
+            "-d tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm -v"
+        )
+
+      * - {
+        // no authentication -> should fail
+
+        val failed = try {
+          CacheFetchTests.check(
+            MavenRepository(
+              s"http://$address"
+            )
+          )
+
+          printErrorMessage()
+          false
+        } catch {
+          case e: Throwable =>
+            true
+        }
+
+        assert(failed)
+      }
+
+      * - {
+        // with authentication -> should work
+
+        try {
+          CacheFetchTests.check(
+            MavenRepository(
+              s"http://$address",
+              authentication = Some(Authentication(user, password))
+            )
+          )
+        } catch {
+          case e: Throwable =>
+            printErrorMessage()
+            throw e
+        }
+      }
+    }
+  }
+
+}

tests/jvm/src/test/scala/coursier/test/CacheFetchTests.scala

@@ -13,55 +13,55 @@ import scala.util.Try
 
 object CacheFetchTests extends TestSuite {
 
-  val tests = TestSuite {
+  def check(extraRepo: Repository): Unit = {
 
-    def check(extraRepo: Repository): Unit = {
+    val tmpDir = Files.createTempDirectory("coursier-cache-fetch-tests").toFile
 
-      val tmpDir = Files.createTempDirectory("coursier-cache-fetch-tests").toFile
+    def cleanTmpDir() = {
+      def delete(f: File): Boolean =
+        if (f.isDirectory) {
+          val removedContent = Option(f.listFiles()).toSeq.flatten.map(delete).forall(x => x)
+          val removedDir = f.delete()
 
-      def cleanTmpDir() = {
+          removedContent && removedDir
-        def delete(f: File): Boolean =
+        } else
-          if (f.isDirectory) {
+          f.delete()
-            val removedContent = Option(f.listFiles()).toSeq.flatten.map(delete).forall(x => x)
-            val removedDir = f.delete()
 
-            removedContent && removedDir
+      if (!delete(tmpDir))
-          } else
+        Console.err.println(s"Warning: unable to remove temporary directory $tmpDir")
-            f.delete()
-
-        if (!delete(tmpDir))
-          Console.err.println(s"Warning: unable to remove temporary directory $tmpDir")
-      }
-      
-      val res = try {
-        val fetch = Fetch.from(
-          Seq(
-            extraRepo,
-            MavenRepository("https://repo1.maven.org/maven2")
-          ),
-          Cache.fetch(
-            tmpDir
-          )
-        )
-
-        val startRes = Resolution(
-          Set(
-            Dependency(
-              Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test"
-            )
-          )
-        )
-
-        startRes.process.run(fetch).run
-      } finally {
-        cleanTmpDir()
-      }
-
-      val errors = res.errors
-
-      assert(errors.isEmpty)
     }
 
+    val res = try {
+      val fetch = Fetch.from(
+        Seq(
+          extraRepo,
+          MavenRepository("https://repo1.maven.org/maven2")
+        ),
+        Cache.fetch(
+          tmpDir
+        )
+      )
+
+      val startRes = Resolution(
+        Set(
+          Dependency(
+            Module("com.github.alexarchambault", "coursier_2.11"), "1.0.0-M9-test"
+          )
+        )
+      )
+
+      startRes.process.run(fetch).run
+    } finally {
+      cleanTmpDir()
+    }
+
+    val errors = res.errors
+
+    assert(errors.isEmpty)
+  }
+
+  val tests = TestSuite {
+
     // using scala-test would allow to put the below comments in the test names...
 
     * - {
@@ -84,64 +84,6 @@ object CacheFetchTests extends TestSuite {
         check(MavenRepository(s"${TestprotocolHandler.protocol}://foo/"))
       }
     }
-
-    'httpAuthentication - {
-      // requires an authenticated HTTP server to be running on localhost:8080 with user 'user'
-      // and password 'pass'
-
-      val address = "localhost:8080"
-      val user = "user"
-      val password = "pass"
-
-      def printErrorMessage() =
-        Console.err.println(
-          Console.RED +
-            s"HTTP authentication tests require a running HTTP server on $address, requiring " +
-            s"basic authentication with user '$user' and password '$password', serving the right " +
-            "files.\n" + Console.RESET +
-            "Run one from the coursier sources with\n" +
-            "  ./coursier launch -r http://dl.bintray.com/scalaz/releases " +
-            "io.get-coursier:simple-web-server_2.11:1.0.0-M12 -- " +
-            "-d tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm -v"
-        )
-
-      * - {
-        // no authentication -> should fail
-
-        val failed = try {
-          check(
-            MavenRepository(
-              s"http://$address"
-            )
-          )
-
-          printErrorMessage()
-          false
-        } catch {
-          case e: Throwable =>
-            true
-        }
-
-        assert(failed)
-      }
-
-      * - {
-        // with authentication -> should work
-
-        try {
-          check(
-            MavenRepository(
-              s"http://$address",
-              authentication = Some(Authentication(user, password))
-            )
-          )
-        } catch {
-          case e: Throwable =>
-            printErrorMessage()
-            throw e
-        }
-      }
-    }
   }
 
 }