diff --git a/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala index 9d4e904cc..4c725813c 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Bootstrap.scala @@ -11,6 +11,8 @@ import caseapp._ import coursier.cli.util.Zip case class Bootstrap( + @Recurse + artifactOptions: ArtifactOptions, @Recurse options: BootstrapOptions ) extends App { @@ -78,6 +80,7 @@ case class Bootstrap( def subFiles0 = helper.fetch( sources = false, javadoc = false, + artifactTypes = artifactOptions.artifactTypes, subset = isolatedDeps.getOrElse(target, Seq.empty).toSet ) @@ -96,11 +99,19 @@ case class Bootstrap( if (options.standalone) ( Seq.empty[String], - helper.fetch(sources = false, javadoc = false) + helper.fetch( + sources = false, + javadoc = false, + artifactTypes = artifactOptions.artifactTypes + ) ) else ( - helper.artifacts(sources = false, javadoc = false).map(_.url), + helper.artifacts( + sources = false, + javadoc = false, + artifactTypes = artifactOptions.artifactTypes + ).map(_.url), Seq.empty[File] ) diff --git a/cli/src/main/scala-2.11/coursier/cli/Fetch.scala b/cli/src/main/scala-2.11/coursier/cli/Fetch.scala index 3677313be..132d1b00e 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Fetch.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Fetch.scala @@ -12,9 +12,13 @@ case class Fetch( options: FetchOptions ) extends App { - val helper = new Helper(options.common, remainingArgs, ignoreErrors = options.force) + val helper = new Helper(options.common, remainingArgs, ignoreErrors = options.artifactOptions.force) - val files0 = helper.fetch(sources = options.sources, javadoc = options.javadoc) + val files0 = helper.fetch( + sources = options.sources, + javadoc = options.javadoc, + artifactTypes = options.artifactOptions.artifactTypes + ) val out = if (options.classpath) diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index e95195d42..c08757f3a 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -492,6 +492,7 @@ class Helper( def artifacts( sources: Boolean, javadoc: Boolean, + artifactTypes: Set[String], subset: Set[Dependency] = null ): Seq[Artifact] = { @@ -514,25 +515,34 @@ class Helper( val res0 = Option(subset).fold(res)(res.subset) - if (classifier0.nonEmpty || sources || javadoc) { - var classifiers = classifier0 - if (sources) - classifiers = classifiers :+ "sources" - if (javadoc) - classifiers = classifiers :+ "javadoc" + val artifacts0 = + if (classifier0.nonEmpty || sources || javadoc) { + var classifiers = classifier0 + if (sources) + classifiers = classifiers :+ "sources" + if (javadoc) + classifiers = classifiers :+ "javadoc" - res0.classifiersArtifacts(classifiers.distinct) - } else - res0.artifacts + res0.dependencyClassifiersArtifacts(classifiers.distinct).map(_._2) + } else + res0.dependencyArtifacts.map(_._2) + + if (artifactTypes("*")) + artifacts0 + else + artifacts0.filter { artifact => + artifactTypes(artifact.`type`) + } } def fetch( sources: Boolean, javadoc: Boolean, + artifactTypes: Set[String], subset: Set[Dependency] = null ): Seq[File] = { - val artifacts0 = artifacts(sources, javadoc, subset) + val artifacts0 = artifacts(sources, javadoc, artifactTypes, subset) val logger = if (verbosityLevel >= 0) @@ -588,7 +598,15 @@ class Helper( lazy val (parentLoader, filteredFiles) = { - val files0 = fetch(sources = false, javadoc = false) + // FIXME That shouldn't be hard-coded this way... + // This whole class ought to be rewritten more cleanly. + val artifactTypes = Set("jar") + + val files0 = fetch( + sources = false, + javadoc = false, + artifactTypes = artifactTypes + ) if (isolated.isolated.isEmpty) (baseLoader, files0) @@ -603,6 +621,7 @@ class Helper( val isolatedFiles = fetch( sources = false, javadoc = false, + artifactTypes = artifactTypes, subset = isolatedDeps.getOrElse(target, Seq.empty).toSet ) diff --git a/cli/src/main/scala-2.11/coursier/cli/Options.scala b/cli/src/main/scala-2.11/coursier/cli/Options.scala index 4d6c803ad..35666de99 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Options.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Options.scala @@ -168,6 +168,33 @@ case class IsolatedLoaderOptions( } +object ArtifactOptions { + def defaultArtifactTypes = Set("jar", "bundle") +} + +case class ArtifactOptions( + @Help("Artifact types that should be retained (e.g. jar, src, doc, etc.) - defaults to jar,bundle") + @Value("type1,type2,...") + @Short("A") + artifactType: List[String], + @Help("Fetch artifacts even if the resolution is errored") + force: Boolean +) { + lazy val artifactTypes = { + val types0 = artifactType + .flatMap(_.split(',')) + .filter(_.nonEmpty) + .toSet + + if (types0.isEmpty) + ArtifactOptions.defaultArtifactTypes + else if (types0("*")) + Set("*") + else + types0 + } +} + case class FetchOptions( @Help("Fetch source artifacts") @Short("S") @@ -178,8 +205,8 @@ case class FetchOptions( @Help("Print java -cp compatible output") @Short("p") classpath: Boolean, - @Help("Fetch artifacts even if the resolution is errored") - force: Boolean, + @Recurse + artifactOptions: ArtifactOptions, @Recurse common: CommonOptions ) @@ -245,6 +272,8 @@ case class SparkSubmitOptions( @Help("Maximum idle time of spark-submit (time with no output). Exit early if no output from spark-submit for more than this duration. Set to 0 for unlimited. (Default: 0)") @Value("seconds") maxIdleTime: Int, + @Recurse + artifactOptions: ArtifactOptions, @Recurse common: CommonOptions ) \ No newline at end of file diff --git a/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala b/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala index 276a7da0b..48cb3376b 100644 --- a/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala +++ b/cli/src/main/scala-2.11/coursier/cli/SparkSubmit.scala @@ -70,8 +70,11 @@ case class SparkSubmit( extraJars = rawExtraJars ) val jars = - helper.fetch(sources = false, javadoc = false) ++ - options.extraJars.map(new File(_)) + helper.fetch( + sources = false, + javadoc = false, + artifactTypes = options.artifactOptions.artifactTypes + ) ++ options.extraJars.map(new File(_)) val (scalaVersion, sparkVersion) = if (options.sparkVersion.isEmpty) @@ -170,6 +173,7 @@ case class SparkSubmit( sparkVersion, options.noDefaultSubmitDependencies, options.submitDependencies.flatMap(_.split(",")).filter(_.nonEmpty), + options.artifactOptions.artifactTypes, options.common ) diff --git a/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala b/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala index 9dc4962ca..d7d53d087 100644 --- a/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala +++ b/cli/src/main/scala-2.11/coursier/cli/spark/Assembly.scala @@ -140,14 +140,15 @@ object Assembly { sparkVersion: String, noDefault: Boolean, extraDependencies: Seq[String], - options: CommonOptions + options: CommonOptions, + artifactTypes: Set[String] = Set("jar") ): Either[String, (File, Seq[File])] = { val base = if (noDefault) Seq() else sparkAssemblyDependencies(scalaVersion, sparkVersion) val helper = new Helper(options, extraDependencies ++ base) - val artifacts = helper.artifacts(sources = false, javadoc = false) - val jars = helper.fetch(sources = false, javadoc = false) + val artifacts = helper.artifacts(sources = false, javadoc = false, artifactTypes = artifactTypes) + val jars = helper.fetch(sources = false, javadoc = false, artifactTypes = artifactTypes) val checksums = artifacts.map { a => val f = a.checksumUrls.get("SHA-1") match { diff --git a/cli/src/main/scala-2.11/coursier/cli/spark/Submit.scala b/cli/src/main/scala-2.11/coursier/cli/spark/Submit.scala index 887dec1af..c3c30f6be 100644 --- a/cli/src/main/scala-2.11/coursier/cli/spark/Submit.scala +++ b/cli/src/main/scala-2.11/coursier/cli/spark/Submit.scala @@ -11,6 +11,7 @@ object Submit { sparkVersion: String, noDefault: Boolean, extraDependencies: Seq[String], + artifactTypes: Set[String], common: CommonOptions ): Seq[File] = { @@ -43,7 +44,11 @@ object Submit { (if (noDefault) Nil else defaultDependencies) ++ extraDependencies ) - helper.fetch(sources = false, javadoc = false) ++ extraCp + helper.fetch( + sources = false, + javadoc = false, + artifactTypes = artifactTypes + ) ++ extraCp } def mainClassName = "org.apache.spark.deploy.SparkSubmit" diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index 4683c55d0..5446e936a 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -197,7 +197,10 @@ final case class Artifact( attributes: Attributes, changing: Boolean, authentication: Option[Authentication] -) +) { + def `type`: String = attributes.`type` + def classifier: String = attributes.classifier +} object Artifact { trait Source { diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index d10a9ae7f..092c4cfac 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -72,10 +72,9 @@ case class IvyRepository( case None => project.publications.collect { case (conf, p) - if (conf == "*" || - conf == dependency.configuration || - project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) - ) && p.classifier.isEmpty => + if conf == "*" || + conf == dependency.configuration || + project.allConfigurations.getOrElse(dependency.configuration, Set.empty).contains(conf) => p } case Some(classifiers) => diff --git a/tests/jvm/src/it/scala/coursier/test/IvyLocalTests.scala b/tests/jvm/src/it/scala/coursier/test/IvyLocalTests.scala index 964f3491b..b035a1363 100644 --- a/tests/jvm/src/it/scala/coursier/test/IvyLocalTests.scala +++ b/tests/jvm/src/it/scala/coursier/test/IvyLocalTests.scala @@ -29,7 +29,7 @@ object IvyLocalTests extends TestSuite { extraRepo = extraRepo )) - val artifacts = res.artifacts.map(_.url) + val artifacts = res.dependencyArtifacts.filter(_._2.`type` == "jar").map(_._2.url) val anyJavadoc = artifacts.exists(_.contains("-javadoc")) val anySources = artifacts.exists(_.contains("-sources")) diff --git a/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml b/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml new file mode 100644 index 000000000..93ca7677f --- /dev/null +++ b/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.md5 b/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.md5 new file mode 100644 index 000000000..ac5f55c5d --- /dev/null +++ b/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.md5 @@ -0,0 +1 @@ +6cc6e8ae02e9c9ba222c05aa5b0ea838 \ No newline at end of file diff --git a/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.sha1 b/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.sha1 new file mode 100644 index 000000000..2e315289d --- /dev/null +++ b/tests/jvm/src/test/resources/test-repo/http/ivy.abc.com/com.example/a_2.11/0.1.0-SNAPSHOT/ivys/ivy.xml.sha1 @@ -0,0 +1 @@ +8b338542251660a1e91c560b77c53dd5870d3ff5 \ No newline at end of file diff --git a/tests/jvm/src/test/scala/coursier/test/IvyTests.scala b/tests/jvm/src/test/scala/coursier/test/IvyTests.scala index 0e6e51c59..be7e3c2a1 100644 --- a/tests/jvm/src/test/scala/coursier/test/IvyTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/IvyTests.scala @@ -1,6 +1,6 @@ package coursier.test -import coursier.Module +import coursier.{ Attributes, Dependency, Module } import coursier.ivy.IvyRepository import utest._ @@ -46,10 +46,56 @@ object IvyTests extends TestSuite { extraRepo = Some(sbtRepo) ) - * - CentralTests.withArtifact(mod, ver, extraRepo = Some(sbtRepo)) { artifact => + * - CentralTests.withArtifact(mod, ver, "jar", extraRepo = Some(sbtRepo)) { artifact => assert(artifact.url == expectedArtifactUrl) } } + + 'testArtifacts - { + + val dep = Dependency( + Module("com.example", "a_2.11"), + "0.1.0-SNAPSHOT", + transitive = false, + attributes = Attributes() + ) + + val repoBase = getClass.getResource("/test-repo/http/ivy.abc.com").toString.stripSuffix("/") + "/" + + val repo = IvyRepository.fromPattern( + repoBase +: coursier.ivy.Pattern.default, + dropInfoAttributes = true + ) + + val mainJarUrl = repoBase + "com.example/a_2.11/0.1.0-SNAPSHOT/jars/a_2.11.jar" + val testJarUrl = repoBase + "com.example/a_2.11/0.1.0-SNAPSHOT/jars/a_2.11-tests.jar" + + * - CentralTests.withArtifacts( + dep = dep, + artifactType = "jar", + extraRepo = Some(repo) + ) { + case Seq(artifact) => + assert(artifact.url == mainJarUrl) + case other => + throw new Exception(s"Unexpected number of artifacts\n${other.mkString("\n")}") + } + + * - CentralTests.withArtifacts( + dep = dep.copy(configuration = "test"), + artifactType = "jar", + extraRepo = Some(repo) + ) { + case Seq(artifact1, artifact2) => + val urls = Set( + artifact1.url, + artifact2.url + ) + assert(urls == Set(mainJarUrl, testJarUrl)) + case other => + throw new Exception(s"Unexpected number of artifacts\n${other.mkString("\n")}") + } + } } } diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index 61ad892ff..b6a3a27bd 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -100,14 +100,12 @@ object CentralTests extends TestSuite { def withArtifact[T]( module: Module, version: String, + artifactType: String, extraRepo: Option[Repository] = None )( f: Artifact => T - ): Future[T] = async { - val dep = Dependency(module, version, transitive = false, attributes = Attributes()) - val res = await(resolve(Set(dep), extraRepo = extraRepo)) - - res.artifacts match { + ): Future[T] = + withArtifacts(module, version, artifactType, extraRepo) { case Seq(artifact) => f(artifact) case other => @@ -116,10 +114,41 @@ object CentralTests extends TestSuite { "Artifacts:\n" + other.map(" " + _).mkString("\n") ) } + + def withArtifacts[T]( + module: Module, + version: String, + artifactType: String, + extraRepo: Option[Repository] = None + )( + f: Seq[Artifact] => T + ): Future[T] = { + val dep = Dependency(module, version, transitive = false, attributes = Attributes()) + withArtifacts(dep, artifactType, extraRepo)(f) } - def ensureArtifactHasExtension(module: Module, version: String, extension: String): Future[Unit] = - withArtifact(module, version) { artifact => + def withArtifacts[T]( + dep: Dependency, + artifactType: String, + extraRepo: Option[Repository] + )( + f: Seq[Artifact] => T + ): Future[T] = async { + val res = await(resolve(Set(dep), extraRepo = extraRepo)) + + assert(res.errors.isEmpty) + assert(res.conflicts.isEmpty) + assert(res.isDone) + + val artifacts = res.dependencyArtifacts.map(_._2).filter { a => + a.`type` == artifactType + } + + f(artifacts) + } + + def ensureHasArtifactWithExtension(module: Module, version: String, artifactType: String, extension: String): Future[Unit] = + withArtifact(module, version, artifactType) { artifact => assert(artifact.url.endsWith("." + extension)) } @@ -275,7 +304,7 @@ object CentralTests extends TestSuite { * - resolutionCheck(mod, version) - * - withArtifact(mod, version) { artifact => + * - withArtifact(mod, version, "jar") { artifact => assert(artifact.url == expectedArtifactUrl) } } @@ -301,27 +330,30 @@ object CentralTests extends TestSuite { 'packaging - { 'aar - { // random aar-based module found on Central - ensureArtifactHasExtension( + ensureHasArtifactWithExtension( Module("com.yandex.android", "speechkit"), "2.5.0", + "aar", "aar" ) } 'bundle - { // has packaging bundle - ensuring coursier gives its artifact the .jar extension - ensureArtifactHasExtension( + ensureHasArtifactWithExtension( Module("com.google.guava", "guava"), "17.0", + "bundle", "jar" ) } 'mavenPlugin - { // has packaging maven-plugin - ensuring coursier gives its artifact the .jar extension - ensureArtifactHasExtension( + ensureHasArtifactWithExtension( Module("org.bytedeco", "javacpp"), "1.1", + "maven-plugin", "jar" ) }