From f5ef7d8179d9f92431c19dfe0affd20866b67416 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 31 May 2017 18:49:48 +0200 Subject: [PATCH] Add non-reg tests for Nexus proxies Ensure everything work fine again with those (things went bad at 1.0.0-RC1, because of the use of directory listings, that may not be exhaustive in proxies - or may be just empty, e.g. currently with nexus 3) --- .travis.yml | 6 + build.sbt | 15 ++ .../main/scala-2.11/coursier/cli/Helper.scala | 41 ++++- .../scala/coursier/core/Definitions.scala | 6 + .../scala/coursier/maven/MavenSource.scala | 148 +++++++++++++----- .../coursier/test/CentralProxyTests.scala | 9 ++ .../src/main/scala/coursier/Tasks.scala | 22 ++- scripts/launch-proxies.sh | 24 +++ scripts/travis.sh | 7 + .../test/scala/coursier/test/MavenTests.scala | 2 +- .../scala/coursier/test/CentralTests.scala | 120 +++++++++----- 11 files changed, 316 insertions(+), 84 deletions(-) create mode 100644 proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala create mode 100755 scripts/launch-proxies.sh diff --git a/.travis.yml b/.travis.yml index a7c87f118..eef4fe69c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ os: - osx script: - scripts/travis.sh +sudo: required +services: + - docker # Uncomment once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed # after_success: # - bash <(curl -s https://codecov.io/bash) @@ -13,6 +16,9 @@ matrix: - env: SCALA_VERSION=2.12.1 PUBLISH=1 os: linux jdk: oraclejdk8 + sudo: required + services: + - docker - env: SCALA_VERSION=2.11.11 PUBLISH=1 os: linux jdk: oraclejdk8 diff --git a/build.sbt b/build.sbt index ea0e113f3..7bb560d5a 100644 --- a/build.sbt +++ b/build.sbt @@ -69,6 +69,19 @@ lazy val tests = crossProject lazy val testsJvm = tests.jvm lazy val testsJs = tests.js +lazy val `proxy-tests` = project + .dependsOn(testsJvm % "test->test") + .configs(Integration) + .settings( + shared, + dontPublish, + hasITs, + coursierPrefix, + libs += Deps.scalaAsync.value, + utest, + sharedTestResources + ) + lazy val cache = project .dependsOn(coreJvm) .settings( @@ -242,6 +255,7 @@ lazy val jvm = project .aggregate( coreJvm, testsJvm, + `proxy-tests`, cache, bootstrap, extra, @@ -297,6 +311,7 @@ lazy val coursier = project `fetch-js`, testsJvm, testsJs, + `proxy-tests`, cache, bootstrap, extra, diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 591726ba3..5d083fdec 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -675,16 +675,47 @@ class Helper( val task = Task.gatherUnordered(tasks) val results = task.unsafePerformSync - val errors = results.collect{case (artifact, -\/(err)) => artifact -> err } - val files0 = results.collect{case (artifact, \/-(f)) => f } + + val (ignoredErrors, errors) = results + .collect { + case (artifact, -\/(err)) => + artifact -> err + } + .partition { + case (a, err) => + val notFound = err match { + case _: FileError.NotFound => true + case _ => false + } + a.isOptional && notFound + } + + val files0 = results.collect { + case (artifact, \/-(f)) => + f + } logger.foreach(_.stop()) + if (verbosityLevel >= 2) + errPrintln( + " Ignoring error(s):\n" + + ignoredErrors + .map { + case (artifact, error) => + s"${artifact.url}: $error" + } + .mkString("\n") + ) + exitIf(errors.nonEmpty) { s" Error:\n" + - errors.map { case (artifact, error) => - s"${artifact.url}: $error" - }.mkString("\n") + errors + .map { + case (artifact, error) => + s"${artifact.url}: $error" + } + .mkString("\n") } files0 diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 0f5dc2d08..dde61eac1 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -198,9 +198,15 @@ final case class Artifact( ) { def `type`: String = attributes.`type` def classifier: String = attributes.classifier + + // TODO make that a proper field after 1.0 (instead of the hack via extra) + def isOptional: Boolean = extra.contains(Artifact.optionalKey) } object Artifact { + + private[coursier] val optionalKey = s"$$optional" + trait Source { def artifacts( dependency: Dependency, diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 8a56db052..83ac196be 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -19,6 +19,13 @@ final case class MavenSource( overrideClassifiers: Option[Seq[String]] ): Seq[Artifact] = { + val packagingTpeMap = project.packagingOpt + .filter(_ != Pom.relocatedPackaging) + .map { packaging => + (MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging + } + .toMap + def artifactOf(publication: Publication) = { val versioning = project @@ -60,38 +67,57 @@ final case class MavenSource( ) } + lazy val defaultPublication = { + + val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type` + + val ext = MavenSource.typeExtension(type0) + + val classifier = + if (dependency.attributes.classifier.isEmpty) + MavenSource.typeDefaultClassifier(type0) + else + dependency.attributes.classifier + + val tpe = packagingTpeMap.getOrElse( + (classifier, ext), + MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext) + ) + + Publication( + dependency.module.name, + tpe, + ext, + classifier + ) + } + overrideClassifiers match { case Some(classifiers) => - classifiers.map { classifier => - Publication( - dependency.module.name, - "jar", - "jar", - classifier - ) - }.map(artifactWithExtra) + classifiers + .map { classifier => + if (classifier == dependency.attributes.classifier) + defaultPublication + else { + val ext = "jar" + val tpe = packagingTpeMap.getOrElse( + (classifier, ext), + MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext) + ) + + Publication( + dependency.module.name, + tpe, + ext, + classifier + ) + } + } + .map(artifactWithExtra) case None => - - val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type` - - val extension = MavenSource.typeExtension(type0) - - val classifier = - if (dependency.attributes.classifier.isEmpty) - MavenSource.typeDefaultClassifier(type0) - else - dependency.attributes.classifier - - Seq( - Publication( - dependency.module.name, - type0, - extension, - classifier - ) - ).map(artifactWithExtra) + Seq(defaultPublication).map(artifactWithExtra) } } @@ -107,12 +133,14 @@ final case class MavenSource( publication: Publication, extra: Map[String, EnrichedPublication] ) { - def artifact: Artifact = { + def artifact: Artifact = + artifact(publication.`type`) + def artifact(versioningType: String): Artifact = { val versioning = project .snapshotVersioning .flatMap(versioning => - mavenVersioning(versioning, publication.classifier, publication.`type`) + mavenVersioning(versioning, publication.classifier, versioningType) ) val path = dependency.module.organization.split('.').toSeq ++ Seq( @@ -123,7 +151,7 @@ final case class MavenSource( val changing0 = changing.getOrElse(project.actualVersion.contains("-SNAPSHOT")) - val extra0 = extra.mapValues(_.artifact).iterator.toMap + val extra0 = extra.mapValues(_.artifact(versioningType)).iterator.toMap Artifact( root + path.mkString("/"), @@ -211,9 +239,10 @@ final case class MavenSource( else if (dependency.attributes.`type`.nonEmpty) enrichedPublications.collect { case p - if p.publication.`type` == dependency.attributes.`type` || - (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow - => + if p.publication.classifier.isEmpty && ( + p.publication.`type` == dependency.attributes.`type` || + (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow + ) => p.artifact } else @@ -226,6 +255,8 @@ final case class MavenSource( res.map(withMetadataExtra) } + private val dummyArtifact = Artifact("", Map(), Map(), Attributes("", ""), changing = false, None) + def artifacts( dependency: Dependency, project: Project, @@ -233,10 +264,53 @@ final case class MavenSource( ): Seq[Artifact] = if (project.packagingOpt.toSeq.contains(Pom.relocatedPackaging)) Nil - else if (project.publications.isEmpty) - artifactsUnknownPublications(dependency, project, overrideClassifiers) - else - artifactsKnownPublications(dependency, project, overrideClassifiers) + else { + + def makeOptional(a: Artifact): Artifact = + a.copy( + extra = a.extra.mapValues(makeOptional).iterator.toMap + (Artifact.optionalKey -> dummyArtifact) + ) + + def merge(a: Artifact, other: Artifact): Artifact = { + + assert(a.url == other.url, s"Merging artifacts with different URLs (${a.url}, ${other.url})") + + val extra = + a.extra.map { + case (k, v) => + k -> other.extra.get(k).fold(v)(merge(v, _)) + } ++ + other.extra + .filterKeys(k => !a.extra.contains(k)) + + a.copy( + checksumUrls = other.checksumUrls ++ a.checksumUrls, + extra = extra + ) + } + + val defaultPublications = artifactsUnknownPublications(dependency, project, overrideClassifiers) + + if (project.publications.isEmpty) + defaultPublications + else { + val listedPublications = artifactsKnownPublications(dependency, project, overrideClassifiers) + val listedUrls = listedPublications.map(_.url).toSet + val defaultPublications0 = defaultPublications.map(makeOptional) + val defaultPublicationsMap = defaultPublications0 + .map(a => a.url -> a) + .toMap + val listedPublications0 = listedPublications.map { a => + defaultPublicationsMap + .get(a.url) + .fold(a)(merge(a, _)) + } + val extraPublications = defaultPublications0 + .filter(a => !listedUrls(a.url)) + + listedPublications0 ++ extraPublications + } + } } object MavenSource { diff --git a/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala b/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala new file mode 100644 index 000000000..8067d2b5a --- /dev/null +++ b/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala @@ -0,0 +1,9 @@ +package coursier.test + +object CentralNexus2ProxyTests extends CentralTests { + override def centralBase = "http://localhost:9081/nexus/content/repositories/central" +} + +object CentralNexus3ProxyTests extends CentralTests { + override def centralBase = "http://localhost:9082/repository/maven-central" +} diff --git a/sbt-coursier/src/main/scala/coursier/Tasks.scala b/sbt-coursier/src/main/scala/coursier/Tasks.scala index 2e22b2df2..494da7a7f 100644 --- a/sbt-coursier/src/main/scala/coursier/Tasks.scala +++ b/sbt-coursier/src/main/scala/coursier/Tasks.scala @@ -1142,13 +1142,23 @@ object Tasks { artifact -> file } - val artifactErrors = artifactFilesOrErrors0.toVector.collect { - case (_, -\/(err)) => - err - } + val (ignoredArtifactErrors, artifactErrors) = artifactFilesOrErrors0 + .toVector + .collect { + case (a, -\/(err)) => + a -> err + } + .partition { + case (a, err) => + val notFound = err match { + case _: FileError.NotFound => true + case _ => false + } + a.isOptional && notFound + } if (artifactErrors.nonEmpty) { - val error = ResolutionError.DownloadErrors(artifactErrors) + val error = ResolutionError.DownloadErrors(artifactErrors.map(_._2)) if (ignoreArtifactErrors) log.warn(error.description(verbosityLevel >= 1)) @@ -1156,7 +1166,7 @@ object Tasks { error.throwException() } - // can be non empty only if ignoreArtifactErrors is true + // can be non empty only if ignoreArtifactErrors is true or some optional artifacts are not found val erroredArtifacts = artifactFilesOrErrors0.collect { case (artifact, -\/(_)) => artifact diff --git a/scripts/launch-proxies.sh b/scripts/launch-proxies.sh new file mode 100755 index 000000000..2170e4f60 --- /dev/null +++ b/scripts/launch-proxies.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +MAX_WAIT=120 + +wait_for() { + TARGET="$1" + I=0 + while ! curl "$TARGET"; do + if [ "$I" -gt "$MAX_WAIT" ]; then + echo "$TARGET not available after $MAX_WAIT seconds" 1>&2 + exit 1 + fi + + I="$(( $I + 1 ))" + sleep 1 + done +} + +docker run -d -p 9081:8081 --name nexus sonatype/nexus:2.14.4 +wait_for "http://localhost:9081/nexus/content/repositories/central/" + +docker run -d -p 9082:8081 --name nexus3 sonatype/nexus3:3.3.1 +wait_for "http://localhost:9082/repository/maven-central/" diff --git a/scripts/travis.sh b/scripts/travis.sh index 2bd4bf256..8e1059c33 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -24,6 +24,10 @@ launchTestRepo() { ./scripts/launch-test-repo.sh "$@" } +launchProxyRepos() { + ./scripts/launch-proxies.sh +} + integrationTestsRequirements() { # Required for ~/.ivy2/local repo tests sbt ++2.11.11 coreJVM/publishLocal @@ -217,6 +221,9 @@ else runSbtShadingTests fi else + # Required for the proxy tests (currently CentralNexus2ProxyTests and CentralNexus3ProxyTests) + launchProxyRepos + runJvmTests testBootstrap diff --git a/tests/jvm/src/test/scala/coursier/test/MavenTests.scala b/tests/jvm/src/test/scala/coursier/test/MavenTests.scala index fd7411606..0e93ca2c3 100644 --- a/tests/jvm/src/test/scala/coursier/test/MavenTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/MavenTests.scala @@ -39,7 +39,7 @@ object MavenTests extends TestSuite { * - CentralTests.withArtifacts( dep = dep, - artifactType = "jar", + artifactType = "src", extraRepo = Some(repo), classifierOpt = Some("sources") ) { diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index 1a6d490c4..03236c46b 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -9,10 +9,16 @@ import coursier.test.compatibility._ import scala.concurrent.Future -object CentralTests extends TestSuite { +object CentralTests extends CentralTests + +abstract class CentralTests extends TestSuite { + + def centralBase = "https://repo1.maven.org/maven2" + + final def isActualCentral = centralBase == "https://repo1.maven.org/maven2" val repositories = Seq[Repository]( - MavenRepository("https://repo1.maven.org/maven2/") + MavenRepository(centralBase) ) def resolve( @@ -347,7 +353,7 @@ object CentralTests extends TestSuite { 'versionFromDependency - { val mod = Module("org.apache.ws.commons", "XmlSchema") val version = "1.1" - val expectedArtifactUrl = "https://repo1.maven.org/maven2/org/apache/ws/commons/XmlSchema/1.1/XmlSchema-1.1.jar" + val expectedArtifactUrl = s"$centralBase/org/apache/ws/commons/XmlSchema/1.1/XmlSchema-1.1.jar" * - resolutionCheck(mod, version) @@ -404,12 +410,27 @@ object CentralTests extends TestSuite { 'packaging - { 'aar - { // random aar-based module found on Central - ensureHasArtifactWithExtension( - Module("com.yandex.android", "speechkit"), - "2.5.0", - "aar", - "aar" + val module = Module("com.yandex.android", "speechkit") + val version = "2.5.0" + val tpe = "aar" + + * - ensureHasArtifactWithExtension( + module, + version, + tpe, + tpe, + attributes = Attributes(tpe) ) + + * - { + if (isActualCentral) + ensureHasArtifactWithExtension( + module, + version, + tpe, + tpe + ) + } } 'bundle - { @@ -570,26 +591,52 @@ object CentralTests extends TestSuite { * - resolutionCheck(mod, version) + val mainTarGzUrl = s"$centralBase/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz" val expectedTarGzArtifactUrls = Set( - "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz", - "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.tar.gz" + mainTarGzUrl, + s"$centralBase/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.tar.gz" ) + val mainZipUrl = s"$centralBase/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip" val expectedZipArtifactUrls = Set( - "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip", - "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.zip" + mainZipUrl, + s"$centralBase/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.zip" ) - * - withArtifacts(mod, version, "tar.gz", classifierOpt = Some("bin"), transitive = true) { artifacts => - assert(artifacts.length == 2) - val urls = artifacts.map(_.url).toSet - assert(urls == expectedTarGzArtifactUrls) + 'tarGz - { + * - { + if (isActualCentral) + withArtifacts(mod, version, "tar.gz", classifierOpt = Some("bin"), transitive = true) { artifacts => + assert(artifacts.length == 2) + val urls = artifacts.map(_.url).toSet + assert(urls == expectedTarGzArtifactUrls) + } + } + * - { + withArtifacts(mod, version, "tar.gz", attributes = Attributes("tar.gz", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts => + assert(artifacts.nonEmpty) + val urls = artifacts.map(_.url).toSet + assert(urls.contains(mainTarGzUrl)) + } + } } - * - withArtifacts(mod, version, "zip", classifierOpt = Some("bin"), transitive = true) { artifacts => - assert(artifacts.length == 2) - val urls = artifacts.map(_.url).toSet - assert(urls == expectedZipArtifactUrls) + 'zip - { + * - { + if (isActualCentral) + withArtifacts(mod, version, "zip", classifierOpt = Some("bin"), transitive = true) { artifacts => + assert(artifacts.length == 2) + val urls = artifacts.map(_.url).toSet + assert(urls == expectedZipArtifactUrls) + } + } + * - { + withArtifacts(mod, version, "zip", attributes = Attributes("zip", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts => + assert(artifacts.nonEmpty) + val urls = artifacts.map(_.url).toSet + assert(urls.contains(mainZipUrl)) + } + } } } @@ -610,7 +657,7 @@ object CentralTests extends TestSuite { val mod = Module("org.apache.commons", "commons-io") val ver = "1.3.2" - val expectedUrl = "https://repo1.maven.org/maven2/commons-io/commons-io/1.3.2/commons-io-1.3.2.jar" + val expectedUrl = s"$centralBase/commons-io/commons-io/1.3.2/commons-io-1.3.2.jar" * - resolutionCheck(mod, ver) @@ -640,8 +687,8 @@ object CentralTests extends TestSuite { val mod = Module("org.yaml", "snakeyaml") val ver = "1.17" - def hasSha1(a: Artifact) = a.extra.contains("SHA-1") - def hasMd5(a: Artifact) = a.extra.contains("MD5") + def hasSha1(a: Artifact) = a.checksumUrls.contains("SHA-1") + def hasMd5(a: Artifact) = a.checksumUrls.contains("MD5") def hasSig(a: Artifact) = a.extra.contains("sig") def sigHasSig(a: Artifact) = a.extra.get("sig").exists(hasSig) @@ -649,24 +696,27 @@ object CentralTests extends TestSuite { * - withArtifacts(mod, ver, "*") { artifacts => - val jarOpt = artifacts.find(_.`type` == "bundle") + val jarOpt = artifacts.find(_.`type` == "bundle").orElse(artifacts.find(_.`type` == "jar")) val pomOpt = artifacts.find(_.`type` == "pom") - if (artifacts.length != 2 || jarOpt.isEmpty || pomOpt.isEmpty) - artifacts.foreach(println) - - assert(artifacts.length == 2) assert(jarOpt.nonEmpty) - assert(pomOpt.nonEmpty) - assert(jarOpt.forall(hasSha1)) - assert(pomOpt.forall(hasSha1)) assert(jarOpt.forall(hasMd5)) - assert(pomOpt.forall(hasMd5)) assert(jarOpt.forall(hasSig)) - assert(pomOpt.forall(hasSig)) - assert(jarOpt.forall(sigHasSig)) - assert(pomOpt.forall(sigHasSig)) + + if (isActualCentral) { + if (artifacts.length != 2 || jarOpt.isEmpty || pomOpt.isEmpty) + artifacts.foreach(println) + + assert(jarOpt.forall(_.`type` == "bundle")) + assert(artifacts.length == 2) + assert(pomOpt.nonEmpty) + assert(pomOpt.forall(hasSha1)) + assert(pomOpt.forall(hasMd5)) + assert(pomOpt.forall(hasSig)) + assert(jarOpt.forall(sigHasSig)) + assert(pomOpt.forall(sigHasSig)) + } } } }