Add non-reg tests for Nexus proxies

Ensure everything work fine again with those (things went bad at
1.0.0-RC1, because of the use of directory listings, that may not be
exhaustive in proxies - or may be just empty, e.g. currently with nexus 3)

.travis.yml

@@ -5,6 +5,9 @@ os:
   - osx
 script:
   - scripts/travis.sh
+sudo: required
+services:
+  - docker
 # Uncomment once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
 # after_success:
 #    - bash <(curl -s https://codecov.io/bash)
@@ -13,6 +16,9 @@ matrix:
     - env: SCALA_VERSION=2.12.1 PUBLISH=1
       os: linux
       jdk: oraclejdk8
+      sudo: required
+      services:
+        - docker
     - env: SCALA_VERSION=2.11.11 PUBLISH=1
       os: linux
       jdk: oraclejdk8

build.sbt

@@ -69,6 +69,19 @@ lazy val tests = crossProject
 lazy val testsJvm = tests.jvm
 lazy val testsJs = tests.js
 
+lazy val `proxy-tests` = project
+  .dependsOn(testsJvm % "test->test")
+  .configs(Integration)
+  .settings(
+    shared,
+    dontPublish,
+    hasITs,
+    coursierPrefix,
+    libs += Deps.scalaAsync.value,
+    utest,
+    sharedTestResources
+  )
+
 lazy val cache = project
   .dependsOn(coreJvm)
   .settings(
@@ -242,6 +255,7 @@ lazy val jvm = project
   .aggregate(
     coreJvm,
     testsJvm,
+    `proxy-tests`,
     cache,
     bootstrap,
     extra,
@@ -297,6 +311,7 @@ lazy val coursier = project
     `fetch-js`,
     testsJvm,
     testsJs,
+    `proxy-tests`,
     cache,
     bootstrap,
     extra,

cli/src/main/scala-2.11/coursier/cli/Helper.scala

@@ -675,16 +675,47 @@ class Helper(
     val task = Task.gatherUnordered(tasks)
 
     val results = task.unsafePerformSync
-    val errors = results.collect{case (artifact, -\/(err)) => artifact -> err }
+
-    val files0 = results.collect{case (artifact, \/-(f)) => f }
+    val (ignoredErrors, errors) = results
+      .collect {
+        case (artifact, -\/(err)) =>
+          artifact -> err
+      }
+      .partition {
+        case (a, err) =>
+          val notFound = err match {
+            case _: FileError.NotFound => true
+            case _ => false
+          }
+          a.isOptional && notFound
+      }
+
+    val files0 = results.collect {
+      case (artifact, \/-(f)) =>
+        f
+    }
 
     logger.foreach(_.stop())
 
+    if (verbosityLevel >= 2)
+      errPrintln(
+        "  Ignoring error(s):\n" +
+        ignoredErrors
+          .map {
+            case (artifact, error) =>
+              s"${artifact.url}: $error"
+          }
+          .mkString("\n")
+      )
+
     exitIf(errors.nonEmpty) {
       s"  Error:\n" +
-      errors.map { case (artifact, error) =>
+      errors
+        .map {
+          case (artifact, error) =>
             s"${artifact.url}: $error"
-      }.mkString("\n")
+        }
+        .mkString("\n")
     }
 
     files0

core/shared/src/main/scala/coursier/core/Definitions.scala

@@ -198,9 +198,15 @@ final case class Artifact(
 ) {
   def `type`: String = attributes.`type`
   def classifier: String = attributes.classifier
+
+  // TODO make that a proper field after 1.0 (instead of the hack via extra)
+  def isOptional: Boolean = extra.contains(Artifact.optionalKey)
 }
 
 object Artifact {
+
+  private[coursier] val optionalKey = s"$$optional"
+
   trait Source {
     def artifacts(
       dependency: Dependency,

core/shared/src/main/scala/coursier/maven/MavenSource.scala

@@ -19,6 +19,13 @@ final case class MavenSource(
     overrideClassifiers: Option[Seq[String]]
   ): Seq[Artifact] = {
 
+    val packagingTpeMap = project.packagingOpt
+      .filter(_ != Pom.relocatedPackaging)
+      .map { packaging =>
+        (MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging
+      }
+      .toMap
+
     def artifactOf(publication: Publication) = {
 
       val versioning = project
@@ -60,23 +67,11 @@ final case class MavenSource(
       )
     }
 
-    overrideClassifiers match {
+    lazy val defaultPublication = {
-      case Some(classifiers) =>
-
-        classifiers.map { classifier =>
-          Publication(
-            dependency.module.name,
-            "jar",
-            "jar",
-            classifier
-          )
-        }.map(artifactWithExtra)
-
-      case None =>
 
       val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type`
 
-        val extension = MavenSource.typeExtension(type0)
+      val ext = MavenSource.typeExtension(type0)
 
       val classifier =
         if (dependency.attributes.classifier.isEmpty)
@@ -84,14 +79,45 @@ final case class MavenSource(
         else
           dependency.attributes.classifier
 
-        Seq(
+      val tpe = packagingTpeMap.getOrElse(
+        (classifier, ext),
+        MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
+      )
+
       Publication(
         dependency.module.name,
-            type0,
+        tpe,
-            extension,
+        ext,
         classifier
       )
-        ).map(artifactWithExtra)
+    }
+
+    overrideClassifiers match {
+      case Some(classifiers) =>
+
+        classifiers
+          .map { classifier =>
+            if (classifier == dependency.attributes.classifier)
+              defaultPublication
+            else {
+              val ext = "jar"
+              val tpe = packagingTpeMap.getOrElse(
+                (classifier, ext),
+                MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
+              )
+
+              Publication(
+                dependency.module.name,
+                tpe,
+                ext,
+                classifier
+              )
+            }
+          }
+          .map(artifactWithExtra)
+
+      case None =>
+        Seq(defaultPublication).map(artifactWithExtra)
     }
   }
 
@@ -107,12 +133,14 @@ final case class MavenSource(
       publication: Publication,
       extra: Map[String, EnrichedPublication]
     ) {
-      def artifact: Artifact = {
+      def artifact: Artifact =
+        artifact(publication.`type`)
+      def artifact(versioningType: String): Artifact = {
 
         val versioning = project
           .snapshotVersioning
           .flatMap(versioning =>
-            mavenVersioning(versioning, publication.classifier, publication.`type`)
+            mavenVersioning(versioning, publication.classifier, versioningType)
           )
 
         val path = dependency.module.organization.split('.').toSeq ++ Seq(
@@ -123,7 +151,7 @@ final case class MavenSource(
 
         val changing0 = changing.getOrElse(project.actualVersion.contains("-SNAPSHOT"))
 
-        val extra0 = extra.mapValues(_.artifact).iterator.toMap
+        val extra0 = extra.mapValues(_.artifact(versioningType)).iterator.toMap
 
         Artifact(
           root + path.mkString("/"),
@@ -211,9 +239,10 @@ final case class MavenSource(
         else if (dependency.attributes.`type`.nonEmpty)
           enrichedPublications.collect {
             case p
-              if p.publication.`type` == dependency.attributes.`type` ||
+              if p.publication.classifier.isEmpty && (
+                   p.publication.`type` == dependency.attributes.`type` ||
                    (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow
-            =>
+                ) =>
               p.artifact
           }
         else
@@ -226,6 +255,8 @@ final case class MavenSource(
     res.map(withMetadataExtra)
   }
 
+  private val dummyArtifact = Artifact("", Map(), Map(), Attributes("", ""), changing = false, None)
+
   def artifacts(
     dependency: Dependency,
     project: Project,
@@ -233,10 +264,53 @@ final case class MavenSource(
   ): Seq[Artifact] =
     if (project.packagingOpt.toSeq.contains(Pom.relocatedPackaging))
       Nil
-    else if (project.publications.isEmpty)
+    else {
-      artifactsUnknownPublications(dependency, project, overrideClassifiers)
+
-    else
+      def makeOptional(a: Artifact): Artifact =
-      artifactsKnownPublications(dependency, project, overrideClassifiers)
+        a.copy(
+          extra = a.extra.mapValues(makeOptional).iterator.toMap + (Artifact.optionalKey -> dummyArtifact)
+        )
+
+      def merge(a: Artifact, other: Artifact): Artifact = {
+
+        assert(a.url == other.url, s"Merging artifacts with different URLs (${a.url}, ${other.url})")
+
+        val extra =
+          a.extra.map {
+            case (k, v) =>
+              k -> other.extra.get(k).fold(v)(merge(v, _))
+          } ++
+          other.extra
+            .filterKeys(k => !a.extra.contains(k))
+
+        a.copy(
+          checksumUrls = other.checksumUrls ++ a.checksumUrls,
+          extra = extra
+        )
+      }
+
+      val defaultPublications = artifactsUnknownPublications(dependency, project, overrideClassifiers)
+
+      if (project.publications.isEmpty)
+        defaultPublications
+      else {
+        val listedPublications = artifactsKnownPublications(dependency, project, overrideClassifiers)
+        val listedUrls = listedPublications.map(_.url).toSet
+        val defaultPublications0 = defaultPublications.map(makeOptional)
+        val defaultPublicationsMap = defaultPublications0
+          .map(a => a.url -> a)
+          .toMap
+        val listedPublications0 = listedPublications.map { a =>
+          defaultPublicationsMap
+            .get(a.url)
+            .fold(a)(merge(a, _))
+        }
+        val extraPublications = defaultPublications0
+          .filter(a => !listedUrls(a.url))
+
+        listedPublications0 ++ extraPublications
+      }
+    }
 }
 
 object MavenSource {

proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala

@@ -0,0 +1,9 @@
+package coursier.test
+
+object CentralNexus2ProxyTests extends CentralTests {
+  override def centralBase = "http://localhost:9081/nexus/content/repositories/central"
+}
+
+object CentralNexus3ProxyTests extends CentralTests {
+  override def centralBase = "http://localhost:9082/repository/maven-central"
+}

sbt-coursier/src/main/scala/coursier/Tasks.scala

@@ -1142,13 +1142,23 @@ object Tasks {
             artifact -> file
         }
 
-        val artifactErrors = artifactFilesOrErrors0.toVector.collect {
+        val (ignoredArtifactErrors, artifactErrors) = artifactFilesOrErrors0
-          case (_, -\/(err)) =>
+          .toVector
-            err
+          .collect {
+            case (a, -\/(err)) =>
+              a -> err
+          }
+          .partition {
+            case (a, err) =>
+              val notFound = err match {
+                case _: FileError.NotFound => true
+                case _ => false
+              }
+              a.isOptional && notFound
           }
 
         if (artifactErrors.nonEmpty) {
-          val error = ResolutionError.DownloadErrors(artifactErrors)
+          val error = ResolutionError.DownloadErrors(artifactErrors.map(_._2))
 
           if (ignoreArtifactErrors)
             log.warn(error.description(verbosityLevel >= 1))
@@ -1156,7 +1166,7 @@ object Tasks {
             error.throwException()
         }
 
-        // can be non empty only if ignoreArtifactErrors is true
+        // can be non empty only if ignoreArtifactErrors is true or some optional artifacts are not found
         val erroredArtifacts = artifactFilesOrErrors0.collect {
           case (artifact, -\/(_)) =>
             artifact

scripts/launch-proxies.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+
+MAX_WAIT=120
+
+wait_for() {
+  TARGET="$1"
+  I=0
+  while ! curl "$TARGET"; do
+    if [ "$I" -gt "$MAX_WAIT" ]; then
+      echo "$TARGET not available after $MAX_WAIT seconds" 1>&2
+      exit 1
+    fi
+
+    I="$(( $I + 1 ))"
+    sleep 1
+  done
+}
+
+docker run -d -p 9081:8081 --name nexus sonatype/nexus:2.14.4
+wait_for "http://localhost:9081/nexus/content/repositories/central/"
+
+docker run -d -p 9082:8081 --name nexus3 sonatype/nexus3:3.3.1
+wait_for "http://localhost:9082/repository/maven-central/"

scripts/travis.sh

@@ -24,6 +24,10 @@ launchTestRepo() {
   ./scripts/launch-test-repo.sh "$@"
 }
 
+launchProxyRepos() {
+  ./scripts/launch-proxies.sh
+}
+
 integrationTestsRequirements() {
   # Required for ~/.ivy2/local repo tests
   sbt ++2.11.11 coreJVM/publishLocal
@@ -217,6 +221,9 @@ else
       runSbtShadingTests
     fi
   else
+    # Required for the proxy tests (currently CentralNexus2ProxyTests and CentralNexus3ProxyTests)
+    launchProxyRepos
+
     runJvmTests
 
     testBootstrap

tests/jvm/src/test/scala/coursier/test/MavenTests.scala

@@ -39,7 +39,7 @@ object MavenTests extends TestSuite {
 
       * - CentralTests.withArtifacts(
         dep = dep,
-        artifactType = "jar",
+        artifactType = "src",
         extraRepo = Some(repo),
         classifierOpt = Some("sources")
       ) {

tests/shared/src/test/scala/coursier/test/CentralTests.scala

@@ -9,10 +9,16 @@ import coursier.test.compatibility._
 
 import scala.concurrent.Future
 
-object CentralTests extends TestSuite {
+object CentralTests extends CentralTests
+
+abstract class CentralTests extends TestSuite {
+
+  def centralBase = "https://repo1.maven.org/maven2"
+
+  final def isActualCentral = centralBase == "https://repo1.maven.org/maven2"
 
   val repositories = Seq[Repository](
-    MavenRepository("https://repo1.maven.org/maven2/")
+    MavenRepository(centralBase)
   )
 
   def resolve(
@@ -347,7 +353,7 @@ object CentralTests extends TestSuite {
     'versionFromDependency - {
       val mod = Module("org.apache.ws.commons", "XmlSchema")
       val version = "1.1"
-      val expectedArtifactUrl = "https://repo1.maven.org/maven2/org/apache/ws/commons/XmlSchema/1.1/XmlSchema-1.1.jar"
+      val expectedArtifactUrl = s"$centralBase/org/apache/ws/commons/XmlSchema/1.1/XmlSchema-1.1.jar"
 
       * - resolutionCheck(mod, version)
 
@@ -404,12 +410,27 @@ object CentralTests extends TestSuite {
     'packaging - {
       'aar - {
         // random aar-based module found on Central
-        ensureHasArtifactWithExtension(
+        val module = Module("com.yandex.android", "speechkit")
-          Module("com.yandex.android", "speechkit"),
+        val version = "2.5.0"
-          "2.5.0",
+        val tpe = "aar"
-          "aar",
+
-          "aar"
+        * - ensureHasArtifactWithExtension(
+          module,
+          version,
+          tpe,
+          tpe,
+          attributes = Attributes(tpe)
         )
+
+        * - {
+          if (isActualCentral)
+            ensureHasArtifactWithExtension(
+              module,
+              version,
+              tpe,
+              tpe
+            )
+        }
       }
 
       'bundle - {
@@ -570,28 +591,54 @@ object CentralTests extends TestSuite {
 
       * - resolutionCheck(mod, version)
 
+      val mainTarGzUrl = s"$centralBase/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz"
       val expectedTarGzArtifactUrls = Set(
-        "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz",
+        mainTarGzUrl,
-        "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.tar.gz"
+        s"$centralBase/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.tar.gz"
       )
 
+      val mainZipUrl = s"$centralBase/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip"
       val expectedZipArtifactUrls = Set(
-        "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip",
+        mainZipUrl,
-        "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.zip"
+        s"$centralBase/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.zip"
       )
 
-      * - withArtifacts(mod, version, "tar.gz", classifierOpt = Some("bin"), transitive = true) { artifacts =>
+      'tarGz - {
+        * - {
+          if (isActualCentral)
+            withArtifacts(mod, version, "tar.gz", classifierOpt = Some("bin"), transitive = true) { artifacts =>
               assert(artifacts.length == 2)
               val urls = artifacts.map(_.url).toSet
               assert(urls == expectedTarGzArtifactUrls)
             }
+        }
+        * - {
+          withArtifacts(mod, version, "tar.gz", attributes = Attributes("tar.gz", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts =>
+            assert(artifacts.nonEmpty)
+            val urls = artifacts.map(_.url).toSet
+            assert(urls.contains(mainTarGzUrl))
+          }
+        }
+      }
 
-      * - withArtifacts(mod, version, "zip", classifierOpt = Some("bin"), transitive = true) { artifacts =>
+      'zip - {
+        * - {
+          if (isActualCentral)
+            withArtifacts(mod, version, "zip", classifierOpt = Some("bin"), transitive = true) { artifacts =>
               assert(artifacts.length == 2)
               val urls = artifacts.map(_.url).toSet
               assert(urls == expectedZipArtifactUrls)
             }
         }
+        * - {
+          withArtifacts(mod, version, "zip", attributes = Attributes("zip", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts =>
+            assert(artifacts.nonEmpty)
+            val urls = artifacts.map(_.url).toSet
+            assert(urls.contains(mainZipUrl))
+          }
+        }
+      }
+    }
 
     'groupIdVersionProperties - {
       resolutionCheck(
@@ -610,7 +657,7 @@ object CentralTests extends TestSuite {
         val mod = Module("org.apache.commons", "commons-io")
         val ver = "1.3.2"
 
-        val expectedUrl = "https://repo1.maven.org/maven2/commons-io/commons-io/1.3.2/commons-io-1.3.2.jar"
+        val expectedUrl = s"$centralBase/commons-io/commons-io/1.3.2/commons-io-1.3.2.jar"
 
         * - resolutionCheck(mod, ver)
 
@@ -640,8 +687,8 @@ object CentralTests extends TestSuite {
       val mod = Module("org.yaml", "snakeyaml")
       val ver = "1.17"
 
-      def hasSha1(a: Artifact) = a.extra.contains("SHA-1")
+      def hasSha1(a: Artifact) = a.checksumUrls.contains("SHA-1")
-      def hasMd5(a: Artifact) = a.extra.contains("MD5")
+      def hasMd5(a: Artifact) = a.checksumUrls.contains("MD5")
       def hasSig(a: Artifact) = a.extra.contains("sig")
       def sigHasSig(a: Artifact) = a.extra.get("sig").exists(hasSig)
 
@@ -649,26 +696,29 @@ object CentralTests extends TestSuite {
 
       * - withArtifacts(mod, ver, "*") { artifacts =>
 
-        val jarOpt = artifacts.find(_.`type` == "bundle")
+        val jarOpt = artifacts.find(_.`type` == "bundle").orElse(artifacts.find(_.`type` == "jar"))
         val pomOpt = artifacts.find(_.`type` == "pom")
 
+        assert(jarOpt.nonEmpty)
+        assert(jarOpt.forall(hasSha1))
+        assert(jarOpt.forall(hasMd5))
+        assert(jarOpt.forall(hasSig))
+
+        if (isActualCentral) {
           if (artifacts.length != 2 || jarOpt.isEmpty || pomOpt.isEmpty)
             artifacts.foreach(println)
 
+          assert(jarOpt.forall(_.`type` == "bundle"))
           assert(artifacts.length == 2)
-        assert(jarOpt.nonEmpty)
           assert(pomOpt.nonEmpty)
-
-        assert(jarOpt.forall(hasSha1))
           assert(pomOpt.forall(hasSha1))
-        assert(jarOpt.forall(hasMd5))
           assert(pomOpt.forall(hasMd5))
-        assert(jarOpt.forall(hasSig))
           assert(pomOpt.forall(hasSig))
           assert(jarOpt.forall(sigHasSig))
           assert(pomOpt.forall(sigHasSig))
         }
       }
     }
+  }
 
 }