Add non-reg tests for Nexus proxies

Ensure everything work fine again with those (things went bad at 1.0.0-RC1, because of the use of directory listings, that may not be exhaustive in proxies - or may be just empty, e.g. currently with nexus 3)
2017-05-31 18:49:48 +02:00 · 2017-05-31 18:49:48 +02:00 · f5ef7d8179
parent 1cd57c071b
commit f5ef7d8179
11 changed files with 316 additions and 84 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -5,6 +5,9 @@ os:
  - osx
 script:
  - scripts/travis.sh
+sudo: required
+services:
+  - docker
 # Uncomment once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
 # after_success:
 #    - bash <(curl -s https://codecov.io/bash)
@ -13,6 +16,9 @@ matrix:
    - env: SCALA_VERSION=2.12.1 PUBLISH=1
      os: linux
      jdk: oraclejdk8
+      sudo: required
+      services:
+        - docker
    - env: SCALA_VERSION=2.11.11 PUBLISH=1
      os: linux
      jdk: oraclejdk8
--- a/build.sbt
+++ b/build.sbt
@ -69,6 +69,19 @@ lazy val tests = crossProject
 lazy val testsJvm = tests.jvm
 lazy val testsJs = tests.js

+lazy val `proxy-tests` = project
+  .dependsOn(testsJvm % "test->test")
+  .configs(Integration)
+  .settings(
+    shared,
+    dontPublish,
+    hasITs,
+    coursierPrefix,
+    libs += Deps.scalaAsync.value,
+    utest,
+    sharedTestResources
+  )
+
 lazy val cache = project
  .dependsOn(coreJvm)
  .settings(
@ -242,6 +255,7 @@ lazy val jvm = project
  .aggregate(
    coreJvm,
    testsJvm,
+    `proxy-tests`,
    cache,
    bootstrap,
    extra,
@ -297,6 +311,7 @@ lazy val coursier = project
    `fetch-js`,
    testsJvm,
    testsJs,
+    `proxy-tests`,
    cache,
    bootstrap,
    extra,
--- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala
+++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala
@ -675,16 +675,47 @@ class Helper(
    val task = Task.gatherUnordered(tasks)

    val results = task.unsafePerformSync
-    val errors = results.collect{case (artifact, -\/(err)) => artifact -> err }
-    val files0 = results.collect{case (artifact, \/-(f)) => f }
+
+    val (ignoredErrors, errors) = results
+      .collect {
+        case (artifact, -\/(err)) =>
+          artifact -> err
+      }
+      .partition {
+        case (a, err) =>
+          val notFound = err match {
+            case _: FileError.NotFound => true
+            case _ => false
+          }
+          a.isOptional && notFound
+      }
+
+    val files0 = results.collect {
+      case (artifact, \/-(f)) =>
+        f
+    }

    logger.foreach(_.stop())

+    if (verbosityLevel >= 2)
+      errPrintln(
+        "  Ignoring error(s):\n" +
+        ignoredErrors
+          .map {
+            case (artifact, error) =>
+              s"${artifact.url}: $error"
+          }
+          .mkString("\n")
+      )
+
    exitIf(errors.nonEmpty) {
      s"  Error:\n" +
-      errors.map { case (artifact, error) =>
-        s"${artifact.url}: $error"
-      }.mkString("\n")
+      errors
+        .map {
+          case (artifact, error) =>
+            s"${artifact.url}: $error"
+        }
+        .mkString("\n")
    }

    files0
--- a/core/shared/src/main/scala/coursier/core/Definitions.scala
+++ b/core/shared/src/main/scala/coursier/core/Definitions.scala
@ -198,9 +198,15 @@ final case class Artifact(
 ) {
  def `type`: String = attributes.`type`
  def classifier: String = attributes.classifier
+
+  // TODO make that a proper field after 1.0 (instead of the hack via extra)
+  def isOptional: Boolean = extra.contains(Artifact.optionalKey)
 }

 object Artifact {
+
+  private[coursier] val optionalKey = s"$$optional"
+
  trait Source {
    def artifacts(
      dependency: Dependency,
--- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala
+++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala
@ -19,6 +19,13 @@ final case class MavenSource(
    overrideClassifiers: Option[Seq[String]]
  ): Seq[Artifact] = {

+    val packagingTpeMap = project.packagingOpt
+      .filter(_ != Pom.relocatedPackaging)
+      .map { packaging =>
+        (MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging
+      }
+      .toMap
+
    def artifactOf(publication: Publication) = {

      val versioning = project
@ -60,38 +67,57 @@ final case class MavenSource(
      )
    }

+    lazy val defaultPublication = {
+
+      val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type`
+
+      val ext = MavenSource.typeExtension(type0)
+
+      val classifier =
+        if (dependency.attributes.classifier.isEmpty)
+          MavenSource.typeDefaultClassifier(type0)
+        else
+          dependency.attributes.classifier
+
+      val tpe = packagingTpeMap.getOrElse(
+        (classifier, ext),
+        MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
+      )
+
+      Publication(
+        dependency.module.name,
+        tpe,
+        ext,
+        classifier
+      )
+    }
+
    overrideClassifiers match {
      case Some(classifiers) =>

-        classifiers.map { classifier =>
-          Publication(
-            dependency.module.name,
-            "jar",
-            "jar",
-            classifier
-          )
-        }.map(artifactWithExtra)
+        classifiers
+          .map { classifier =>
+            if (classifier == dependency.attributes.classifier)
+              defaultPublication
+            else {
+              val ext = "jar"
+              val tpe = packagingTpeMap.getOrElse(
+                (classifier, ext),
+                MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
+              )
+
+              Publication(
+                dependency.module.name,
+                tpe,
+                ext,
+                classifier
+              )
+            }
+          }
+          .map(artifactWithExtra)

      case None =>
-
-        val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type`
-
-        val extension = MavenSource.typeExtension(type0)
-
-        val classifier =
-          if (dependency.attributes.classifier.isEmpty)
-            MavenSource.typeDefaultClassifier(type0)
-          else
-            dependency.attributes.classifier
-
-        Seq(
-          Publication(
-            dependency.module.name,
-            type0,
-            extension,
-            classifier
-          )
-        ).map(artifactWithExtra)
+        Seq(defaultPublication).map(artifactWithExtra)
    }
  }

@ -107,12 +133,14 @@ final case class MavenSource(
      publication: Publication,
      extra: Map[String, EnrichedPublication]
    ) {
-      def artifact: Artifact = {
+      def artifact: Artifact =
+        artifact(publication.`type`)
+      def artifact(versioningType: String): Artifact = {

        val versioning = project
          .snapshotVersioning
          .flatMap(versioning =>
-            mavenVersioning(versioning, publication.classifier, publication.`type`)
+            mavenVersioning(versioning, publication.classifier, versioningType)
          )

        val path = dependency.module.organization.split('.').toSeq ++ Seq(
@ -123,7 +151,7 @@ final case class MavenSource(

        val changing0 = changing.getOrElse(project.actualVersion.contains("-SNAPSHOT"))

-        val extra0 = extra.mapValues(_.artifact).iterator.toMap
+        val extra0 = extra.mapValues(_.artifact(versioningType)).iterator.toMap

        Artifact(
          root + path.mkString("/"),
@ -211,9 +239,10 @@ final case class MavenSource(
        else if (dependency.attributes.`type`.nonEmpty)
          enrichedPublications.collect {
            case p
-              if p.publication.`type` == dependency.attributes.`type` ||
-                (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow
-            =>
+              if p.publication.classifier.isEmpty && (
+                   p.publication.`type` == dependency.attributes.`type` ||
+                   (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow
+                ) =>
              p.artifact
          }
        else
@ -226,6 +255,8 @@ final case class MavenSource(
    res.map(withMetadataExtra)
  }

+  private val dummyArtifact = Artifact("", Map(), Map(), Attributes("", ""), changing = false, None)
+
  def artifacts(
    dependency: Dependency,
    project: Project,
@ -233,10 +264,53 @@ final case class MavenSource(
  ): Seq[Artifact] =
    if (project.packagingOpt.toSeq.contains(Pom.relocatedPackaging))
      Nil
-    else if (project.publications.isEmpty)
-      artifactsUnknownPublications(dependency, project, overrideClassifiers)
-    else
-      artifactsKnownPublications(dependency, project, overrideClassifiers)
+    else {
+
+      def makeOptional(a: Artifact): Artifact =
+        a.copy(
+          extra = a.extra.mapValues(makeOptional).iterator.toMap + (Artifact.optionalKey -> dummyArtifact)
+        )
+
+      def merge(a: Artifact, other: Artifact): Artifact = {
+
+        assert(a.url == other.url, s"Merging artifacts with different URLs (${a.url}, ${other.url})")
+
+        val extra =
+          a.extra.map {
+            case (k, v) =>
+              k -> other.extra.get(k).fold(v)(merge(v, _))
+          } ++
+          other.extra
+            .filterKeys(k => !a.extra.contains(k))
+
+        a.copy(
+          checksumUrls = other.checksumUrls ++ a.checksumUrls,
+          extra = extra
+        )
+      }
+
+      val defaultPublications = artifactsUnknownPublications(dependency, project, overrideClassifiers)
+
+      if (project.publications.isEmpty)
+        defaultPublications
+      else {
+        val listedPublications = artifactsKnownPublications(dependency, project, overrideClassifiers)
+        val listedUrls = listedPublications.map(_.url).toSet
+        val defaultPublications0 = defaultPublications.map(makeOptional)
+        val defaultPublicationsMap = defaultPublications0
+          .map(a => a.url -> a)
+          .toMap
+        val listedPublications0 = listedPublications.map { a =>
+          defaultPublicationsMap
+            .get(a.url)
+            .fold(a)(merge(a, _))
+        }
+        val extraPublications = defaultPublications0
+          .filter(a => !listedUrls(a.url))
+
+        listedPublications0 ++ extraPublications
+      }
+    }
 }

 object MavenSource {
--- a/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala
+++ b/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala
@ -0,0 +1,9 @@
+package coursier.test
+
+object CentralNexus2ProxyTests extends CentralTests {
+  override def centralBase = "http://localhost:9081/nexus/content/repositories/central"
+}
+
+object CentralNexus3ProxyTests extends CentralTests {
+  override def centralBase = "http://localhost:9082/repository/maven-central"
+}
--- a/sbt-coursier/src/main/scala/coursier/Tasks.scala
+++ b/sbt-coursier/src/main/scala/coursier/Tasks.scala
@ -1142,13 +1142,23 @@ object Tasks {
            artifact -> file
        }

-        val artifactErrors = artifactFilesOrErrors0.toVector.collect {
-          case (_, -\/(err)) =>
-            err
-        }
+        val (ignoredArtifactErrors, artifactErrors) = artifactFilesOrErrors0
+          .toVector
+          .collect {
+            case (a, -\/(err)) =>
+              a -> err
+          }
+          .partition {
+            case (a, err) =>
+              val notFound = err match {
+                case _: FileError.NotFound => true
+                case _ => false
+              }
+              a.isOptional && notFound
+          }

        if (artifactErrors.nonEmpty) {
-          val error = ResolutionError.DownloadErrors(artifactErrors)
+          val error = ResolutionError.DownloadErrors(artifactErrors.map(_._2))

          if (ignoreArtifactErrors)
            log.warn(error.description(verbosityLevel >= 1))
@ -1156,7 +1166,7 @@ object Tasks {
            error.throwException()
        }

-        // can be non empty only if ignoreArtifactErrors is true
+        // can be non empty only if ignoreArtifactErrors is true or some optional artifacts are not found
        val erroredArtifacts = artifactFilesOrErrors0.collect {
          case (artifact, -\/(_)) =>
            artifact
--- a/scripts/launch-proxies.sh
+++ b/scripts/launch-proxies.sh
@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+
+MAX_WAIT=120
+
+wait_for() {
+  TARGET="$1"
+  I=0
+  while ! curl "$TARGET"; do
+    if [ "$I" -gt "$MAX_WAIT" ]; then
+      echo "$TARGET not available after $MAX_WAIT seconds" 1>&2
+      exit 1
+    fi
+
+    I="$(( $I + 1 ))"
+    sleep 1
+  done
+}
+
+docker run -d -p 9081:8081 --name nexus sonatype/nexus:2.14.4
+wait_for "http://localhost:9081/nexus/content/repositories/central/"
+
+docker run -d -p 9082:8081 --name nexus3 sonatype/nexus3:3.3.1
+wait_for "http://localhost:9082/repository/maven-central/"
--- a/scripts/travis.sh
+++ b/scripts/travis.sh
@ -24,6 +24,10 @@ launchTestRepo() {
  ./scripts/launch-test-repo.sh "$@"
 }

+launchProxyRepos() {
+  ./scripts/launch-proxies.sh
+}
+
 integrationTestsRequirements() {
  # Required for ~/.ivy2/local repo tests
  sbt ++2.11.11 coreJVM/publishLocal
@ -217,6 +221,9 @@ else
      runSbtShadingTests
    fi
  else
+    # Required for the proxy tests (currently CentralNexus2ProxyTests and CentralNexus3ProxyTests)
+    launchProxyRepos
+
    runJvmTests

    testBootstrap
--- a/tests/jvm/src/test/scala/coursier/test/MavenTests.scala
+++ b/tests/jvm/src/test/scala/coursier/test/MavenTests.scala
@ -39,7 +39,7 @@ object MavenTests extends TestSuite {

      * - CentralTests.withArtifacts(
        dep = dep,
-        artifactType = "jar",
+        artifactType = "src",
        extraRepo = Some(repo),
        classifierOpt = Some("sources")
      ) {
--- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala
+++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala
@ -9,10 +9,16 @@ import coursier.test.compatibility._

 import scala.concurrent.Future

-object CentralTests extends TestSuite {
+object CentralTests extends CentralTests
+
+abstract class CentralTests extends TestSuite {
+
+  def centralBase = "https://repo1.maven.org/maven2"
+
+  final def isActualCentral = centralBase == "https://repo1.maven.org/maven2"

  val repositories = Seq[Repository](
-    MavenRepository("https://repo1.maven.org/maven2/")
+    MavenRepository(centralBase)
  )

  def resolve(
@ -347,7 +353,7 @@ object CentralTests extends TestSuite {
    'versionFromDependency - {
      val mod = Module("org.apache.ws.commons", "XmlSchema")
      val version = "1.1"
-      val expectedArtifactUrl = "https://repo1.maven.org/maven2/org/apache/ws/commons/XmlSchema/1.1/XmlSchema-1.1.jar"
+      val expectedArtifactUrl = s"$centralBase/org/apache/ws/commons/XmlSchema/1.1/XmlSchema-1.1.jar"

      * - resolutionCheck(mod, version)

@ -404,12 +410,27 @@ object CentralTests extends TestSuite {
    'packaging - {
      'aar - {
        // random aar-based module found on Central
-        ensureHasArtifactWithExtension(
-          Module("com.yandex.android", "speechkit"),
-          "2.5.0",
-          "aar",
-          "aar"
+        val module = Module("com.yandex.android", "speechkit")
+        val version = "2.5.0"
+        val tpe = "aar"
+
+        * - ensureHasArtifactWithExtension(
+          module,
+          version,
+          tpe,
+          tpe,
+          attributes = Attributes(tpe)
        )
+
+        * - {
+          if (isActualCentral)
+            ensureHasArtifactWithExtension(
+              module,
+              version,
+              tpe,
+              tpe
+            )
+        }
      }

      'bundle - {
@ -570,26 +591,52 @@ object CentralTests extends TestSuite {

      * - resolutionCheck(mod, version)

+      val mainTarGzUrl = s"$centralBase/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz"
      val expectedTarGzArtifactUrls = Set(
-        "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz",
-        "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.tar.gz"
+        mainTarGzUrl,
+        s"$centralBase/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.tar.gz"
      )

+      val mainZipUrl = s"$centralBase/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip"
      val expectedZipArtifactUrls = Set(
-        "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip",
-        "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.zip"
+        mainZipUrl,
+        s"$centralBase/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.zip"
      )

-      * - withArtifacts(mod, version, "tar.gz", classifierOpt = Some("bin"), transitive = true) { artifacts =>
-        assert(artifacts.length == 2)
-        val urls = artifacts.map(_.url).toSet
-        assert(urls == expectedTarGzArtifactUrls)
+      'tarGz - {
+        * - {
+          if (isActualCentral)
+            withArtifacts(mod, version, "tar.gz", classifierOpt = Some("bin"), transitive = true) { artifacts =>
+              assert(artifacts.length == 2)
+              val urls = artifacts.map(_.url).toSet
+              assert(urls == expectedTarGzArtifactUrls)
+            }
+        }
+        * - {
+          withArtifacts(mod, version, "tar.gz", attributes = Attributes("tar.gz", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts =>
+            assert(artifacts.nonEmpty)
+            val urls = artifacts.map(_.url).toSet
+            assert(urls.contains(mainTarGzUrl))
+          }
+        }
      }

-      * - withArtifacts(mod, version, "zip", classifierOpt = Some("bin"), transitive = true) { artifacts =>
-        assert(artifacts.length == 2)
-        val urls = artifacts.map(_.url).toSet
-        assert(urls == expectedZipArtifactUrls)
+      'zip - {
+        * - {
+          if (isActualCentral)
+            withArtifacts(mod, version, "zip", classifierOpt = Some("bin"), transitive = true) { artifacts =>
+              assert(artifacts.length == 2)
+              val urls = artifacts.map(_.url).toSet
+              assert(urls == expectedZipArtifactUrls)
+            }
+        }
+        * - {
+          withArtifacts(mod, version, "zip", attributes = Attributes("zip", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts =>
+            assert(artifacts.nonEmpty)
+            val urls = artifacts.map(_.url).toSet
+            assert(urls.contains(mainZipUrl))
+          }
+        }
      }
    }

@ -610,7 +657,7 @@ object CentralTests extends TestSuite {
        val mod = Module("org.apache.commons", "commons-io")
        val ver = "1.3.2"

-        val expectedUrl = "https://repo1.maven.org/maven2/commons-io/commons-io/1.3.2/commons-io-1.3.2.jar"
+        val expectedUrl = s"$centralBase/commons-io/commons-io/1.3.2/commons-io-1.3.2.jar"

        * - resolutionCheck(mod, ver)

@ -640,8 +687,8 @@ object CentralTests extends TestSuite {
      val mod = Module("org.yaml", "snakeyaml")
      val ver = "1.17"

-      def hasSha1(a: Artifact) = a.extra.contains("SHA-1")
-      def hasMd5(a: Artifact) = a.extra.contains("MD5")
+      def hasSha1(a: Artifact) = a.checksumUrls.contains("SHA-1")
+      def hasMd5(a: Artifact) = a.checksumUrls.contains("MD5")
      def hasSig(a: Artifact) = a.extra.contains("sig")
      def sigHasSig(a: Artifact) = a.extra.get("sig").exists(hasSig)

@ -649,24 +696,27 @@ object CentralTests extends TestSuite {

      * - withArtifacts(mod, ver, "*") { artifacts =>

-        val jarOpt = artifacts.find(_.`type` == "bundle")
+        val jarOpt = artifacts.find(_.`type` == "bundle").orElse(artifacts.find(_.`type` == "jar"))
        val pomOpt = artifacts.find(_.`type` == "pom")

-        if (artifacts.length != 2 || jarOpt.isEmpty || pomOpt.isEmpty)
-          artifacts.foreach(println)
-
-        assert(artifacts.length == 2)
        assert(jarOpt.nonEmpty)
-        assert(pomOpt.nonEmpty)
-
        assert(jarOpt.forall(hasSha1))
-        assert(pomOpt.forall(hasSha1))
        assert(jarOpt.forall(hasMd5))
-        assert(pomOpt.forall(hasMd5))
        assert(jarOpt.forall(hasSig))
-        assert(pomOpt.forall(hasSig))
-        assert(jarOpt.forall(sigHasSig))
-        assert(pomOpt.forall(sigHasSig))
+
+        if (isActualCentral) {
+          if (artifacts.length != 2 || jarOpt.isEmpty || pomOpt.isEmpty)
+            artifacts.foreach(println)
+
+          assert(jarOpt.forall(_.`type` == "bundle"))
+          assert(artifacts.length == 2)
+          assert(pomOpt.nonEmpty)
+          assert(pomOpt.forall(hasSha1))
+          assert(pomOpt.forall(hasMd5))
+          assert(pomOpt.forall(hasSig))
+          assert(jarOpt.forall(sigHasSig))
+          assert(pomOpt.forall(sigHasSig))
+        }
      }
    }
  }