From f8884d4170243063e6b52870dc554312395042b7 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 16 Mar 2018 13:44:43 +0100 Subject: [PATCH 1/3] Re-enable Nexus proxy tests, launch the mirrors from the tests --- build.sbt | 5 +- project/Deps.scala | 1 + .../test/CentralNexus2ProxyTests.scala | 11 +++ .../test/CentralNexus3ProxyTests.scala | 15 +++ .../coursier/test/CentralProxyTests.scala | 11 --- .../it/scala/coursier/test/NexusDocker.scala | 96 +++++++++++++++++++ scripts/launch-proxies.sh | 24 ----- scripts/travis.sh | 9 -- .../scala/coursier/test/CentralTests.scala | 2 +- 9 files changed, 128 insertions(+), 46 deletions(-) create mode 100644 proxy-tests/src/it/scala/coursier/test/CentralNexus2ProxyTests.scala create mode 100644 proxy-tests/src/it/scala/coursier/test/CentralNexus3ProxyTests.scala delete mode 100644 proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala create mode 100644 proxy-tests/src/it/scala/coursier/test/NexusDocker.scala delete mode 100755 scripts/launch-proxies.sh diff --git a/build.sbt b/build.sbt index 03eb4008c..01ac32f7b 100644 --- a/build.sbt +++ b/build.sbt @@ -67,7 +67,10 @@ lazy val `proxy-tests` = project dontPublish, hasITs, coursierPrefix, - libs += Deps.scalaAsync.value, + libs ++= Seq( + Deps.dockerClient, + Deps.scalaAsync.value + ), utest, sharedTestResources ) diff --git a/project/Deps.scala b/project/Deps.scala index 6dbbde0c0..867904dd4 100644 --- a/project/Deps.scala +++ b/project/Deps.scala @@ -16,6 +16,7 @@ object Deps { def jackson = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.8.4" def scalatest = "org.scalatest" %% "scalatest" % "3.0.0" def junit = "junit" % "junit" % "4.12" + def dockerClient = "com.spotify" % "docker-client" % "8.11.1" def sbtPgp = Def.setting { val sbtv = CrossVersion.binarySbtVersion(sbtVersion.in(pluginCrossBuild).value) diff --git a/proxy-tests/src/it/scala/coursier/test/CentralNexus2ProxyTests.scala b/proxy-tests/src/it/scala/coursier/test/CentralNexus2ProxyTests.scala new file mode 100644 index 000000000..385efffc2 --- /dev/null +++ b/proxy-tests/src/it/scala/coursier/test/CentralNexus2ProxyTests.scala @@ -0,0 +1,11 @@ +package coursier.test + +object CentralNexus2ProxyTests extends CentralTests { + + val repo = NexusDocker("sonatype/nexus:2.14.4", "nexus/content/repositories/central", 9081) + + override def utestAfterAll(): Unit = + repo.shutdown() + + override def centralBase = repo.base +} diff --git a/proxy-tests/src/it/scala/coursier/test/CentralNexus3ProxyTests.scala b/proxy-tests/src/it/scala/coursier/test/CentralNexus3ProxyTests.scala new file mode 100644 index 000000000..fce4a5eb7 --- /dev/null +++ b/proxy-tests/src/it/scala/coursier/test/CentralNexus3ProxyTests.scala @@ -0,0 +1,15 @@ +package coursier.test + +object CentralNexus3ProxyTests extends CentralTests { + + val repo = NexusDocker( + "sonatype/nexus3:3.3.1", + "repository/maven-central/", // 400 error without the trailing '/' + 9082 + ) + + override def utestAfterAll(): Unit = + repo.shutdown() + + override def centralBase = repo.base.stripSuffix("/") +} diff --git a/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala b/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala deleted file mode 100644 index 485c6b646..000000000 --- a/proxy-tests/src/it/scala/coursier/test/CentralProxyTests.scala +++ /dev/null @@ -1,11 +0,0 @@ -// FIXME: https://github.com/coursier/coursier/issues/752 -// package coursier.test -// -// object CentralNexus2ProxyTests extends CentralTests { -// override def centralBase = "http://localhost:9081/nexus/content/repositories/central" -// } -// -// object CentralNexus3ProxyTests extends CentralTests { -// override def centralBase = "http://localhost:9082/repository/maven-central" -// } - diff --git a/proxy-tests/src/it/scala/coursier/test/NexusDocker.scala b/proxy-tests/src/it/scala/coursier/test/NexusDocker.scala new file mode 100644 index 000000000..735269ac7 --- /dev/null +++ b/proxy-tests/src/it/scala/coursier/test/NexusDocker.scala @@ -0,0 +1,96 @@ +package coursier.test + +import java.io.InputStream + +import com.spotify.docker.client.DefaultDockerClient +import com.spotify.docker.client.messages.{ContainerConfig, HostConfig, PortBinding} +import coursier.internal.FileUtil + +import scala.annotation.tailrec +import scala.collection.JavaConverters._ +import scala.concurrent.duration.DurationInt +import scala.util.Try + +final case class NexusDocker(base: String, shutdown: () => Unit) + +object NexusDocker { + def apply( + image: String, + basePath: String, + // can't find a way to get back a randomly assigned port (even following https://github.com/spotify/docker-client/issues/625) + // so that one has to be specified + hostPort: Int + ): NexusDocker = { + + val addr = s"localhost:$hostPort" + + def log(s: String): Unit = + Console.err.println(s"[$image @ $addr] $s") + + val docker = DefaultDockerClient.fromEnv().build() + docker.pull(image) + + val portBindings = Map("8081" -> Seq(PortBinding.of("0.0.0.0", hostPort)).asJava) + + val hostConfig = HostConfig.builder().portBindings(portBindings.asJava).build() + + val containerConfig = ContainerConfig.builder() + .hostConfig(hostConfig) + .image(image) + .exposedPorts(portBindings.keys.toSeq: _*) + .build() + + var idOpt = Option.empty[String] + + def shutdown(): Unit = + for (id <- idOpt) { + Try(docker.killContainer(id)) + docker.removeContainer(id) + docker.close() + } + + try { + val creation = docker.createContainer(containerConfig) + + val id = creation.id() + idOpt = Some(id) + + log(s"starting container $id") + docker.startContainer(id) + + val base: String = + s"http://localhost:$hostPort/$basePath" + + log(s"waiting for nexus server to be up-and-running") + + val retryDuration = 2.seconds + + @tailrec + def loop(retry: Int): Unit = + if (retry > 0) { + val url = new java.net.URL(base) + var is: InputStream = null + try { + is = url.openStream() + FileUtil.readFully(is) + log("nexus up") + } catch { + case e: java.io.IOException => + if (is != null) is.close() + log(s"Caught $e, retrying in $retryDuration") + Thread.sleep(retryDuration.toMillis) + loop(retry - 1) + } + } else + throw new Exception(s"Timeout when waiting for container for $image to be up-and-running") + + loop(60) + + NexusDocker(base, () => shutdown()) + } catch { + case t: Throwable => + shutdown() + throw t + } + } +} diff --git a/scripts/launch-proxies.sh b/scripts/launch-proxies.sh deleted file mode 100755 index 2170e4f60..000000000 --- a/scripts/launch-proxies.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -e - -MAX_WAIT=120 - -wait_for() { - TARGET="$1" - I=0 - while ! curl "$TARGET"; do - if [ "$I" -gt "$MAX_WAIT" ]; then - echo "$TARGET not available after $MAX_WAIT seconds" 1>&2 - exit 1 - fi - - I="$(( $I + 1 ))" - sleep 1 - done -} - -docker run -d -p 9081:8081 --name nexus sonatype/nexus:2.14.4 -wait_for "http://localhost:9081/nexus/content/repositories/central/" - -docker run -d -p 9082:8081 --name nexus3 sonatype/nexus3:3.3.1 -wait_for "http://localhost:9082/repository/maven-central/" diff --git a/scripts/travis.sh b/scripts/travis.sh index 643388e59..51939af71 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -39,12 +39,6 @@ launchTestRepo() { ./scripts/launch-test-repo.sh "$@" } -launchProxyRepos() { - if [ "$(uname)" != "Darwin" ]; then - ./scripts/launch-proxies.sh - fi -} - integrationTestsRequirements() { # Required for ~/.ivy2/local repo tests sbt ++2.11.12 coreJVM/publishLocal ++2.12.4 cli/publishLocal @@ -189,9 +183,6 @@ else runSbtShadingTests fi else - # Required for the proxy tests (currently CentralNexus2ProxyTests and CentralNexus3ProxyTests) - launchProxyRepos - runJvmTests testBootstrap diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index 2a57aab2f..b46b7e164 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -18,7 +18,7 @@ abstract class CentralTests extends TestSuite { final def isActualCentral = centralBase == "https://repo1.maven.org/maven2" - val repositories = Seq[Repository]( + lazy val repositories = Seq[Repository]( MavenRepository(centralBase) ) From 86d3b923e48d0b0113a9ea64976d04e9714ced1e Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 16 Mar 2018 13:44:43 +0100 Subject: [PATCH 2/3] Disable test whose dependencies can change, when using proxies --- .../src/test/scala/coursier/test/CentralTests.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index b46b7e164..b82324821 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -371,12 +371,13 @@ abstract class CentralTests extends TestSuite { } 'versionInterval - { - // Warning: needs to be updated when new versions of org.webjars.bower:jquery and - // org.webjars.bower:jquery-mousewheel are published :-| - resolutionCheck( - Module("org.webjars.bower", "malihu-custom-scrollbar-plugin"), - "3.1.5" - ) + if (isActualCentral) + resolutionCheck( + Module("org.webjars.bower", "malihu-custom-scrollbar-plugin"), + "3.1.5" + ) + else + Future.successful(()) } 'latestRevision - { From 21d550543b903e92852a387ef12ea0637f2918d9 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 16 Mar 2018 13:44:44 +0100 Subject: [PATCH 3/3] Ensure a Future is passed back to utest in all cases --- .../test/scala/coursier/test/CentralTests.scala | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index b82324821..497dec1b3 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -63,7 +63,7 @@ abstract class CentralTests extends TestSuite { extraRepos: Seq[Repository] = Nil, configuration: String = "", profiles: Option[Set[String]] = None - ) = + ): Future[Unit] = async { val attrPathPart = if (module.attributes.isEmpty) @@ -722,6 +722,8 @@ abstract class CentralTests extends TestSuite { val urls = artifacts.map(_.url).toSet assert(urls == expectedTarGzArtifactUrls) } + else + Future.successful(()) } * - { withArtifacts(mod, version, "tar.gz", attributes = Attributes("tar.gz", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts => @@ -740,6 +742,8 @@ abstract class CentralTests extends TestSuite { val urls = artifacts.map(_.url).toSet assert(urls == expectedZipArtifactUrls) } + else + Future.successful(()) } * - { withArtifacts(mod, version, "zip", attributes = Attributes("zip", "bin"), classifierOpt = Some("bin"), transitive = true) { artifacts => @@ -838,6 +842,8 @@ abstract class CentralTests extends TestSuite { * - { if (isActualCentral) // doesn't work via proxies, which don't list all the upstream available versions resolutionCheck(mod, ver) + else + Future.successful(()) } } @@ -848,6 +854,8 @@ abstract class CentralTests extends TestSuite { * - { if (isActualCentral) // if false, the tests rely on things straight from Central, which can be updated sometimes… resolutionCheck(mod, ver) + else + Future.successful(()) } } @@ -873,6 +881,8 @@ abstract class CentralTests extends TestSuite { assert(mainArtifactOpt.nonEmpty) assert(mainArtifactOpt.forall(_.isOptional)) } + else + Future.successful(()) } * - withArtifacts(mod, ver, "jar", optional = false) { artifacts => @@ -895,6 +905,8 @@ abstract class CentralTests extends TestSuite { ) assert(urls == expectedUrls) } + else + Future.successful(()) } } @@ -969,6 +981,8 @@ abstract class CentralTests extends TestSuite { assert(expectedUrls.forall(urls)) } + else + Future.successful(()) } } }