From ef21746c812466140786f465506dd227ddf14c6c Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 11 Apr 2017 14:38:20 +0200 Subject: [PATCH 1/2] Be fine with extensions / types with dots Like tar.gz --- .../coursier/maven/MavenRepository.scala | 2 +- .../test/scala/coursier/test/IvyTests.scala | 6 ++- .../org.apache.maven/apache-maven/3.3.9 | 44 +++++++++++++++++ .../scala/coursier/test/CentralTests.scala | 48 ++++++++++++++++--- 4 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 tests/shared/src/test/resources/resolutions/org.apache.maven/apache-maven/3.3.9 diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index d92d0b45a..680d69a36 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -266,7 +266,7 @@ final case class MavenRepository( // TODO There should be a regex for that... if (fileName.startsWith(prefix)) { val end = fileName.stripPrefix(prefix) - val idx = end.lastIndexOf('.') + val idx = end.indexOf('.') if (idx >= 0) { val ext = end.drop(idx + 1) val rem = end.take(idx) diff --git a/tests/jvm/src/test/scala/coursier/test/IvyTests.scala b/tests/jvm/src/test/scala/coursier/test/IvyTests.scala index be7e3c2a1..e6c5e3192 100644 --- a/tests/jvm/src/test/scala/coursier/test/IvyTests.scala +++ b/tests/jvm/src/test/scala/coursier/test/IvyTests.scala @@ -73,7 +73,8 @@ object IvyTests extends TestSuite { * - CentralTests.withArtifacts( dep = dep, artifactType = "jar", - extraRepo = Some(repo) + extraRepo = Some(repo), + classifierOpt = None ) { case Seq(artifact) => assert(artifact.url == mainJarUrl) @@ -84,7 +85,8 @@ object IvyTests extends TestSuite { * - CentralTests.withArtifacts( dep = dep.copy(configuration = "test"), artifactType = "jar", - extraRepo = Some(repo) + extraRepo = Some(repo), + classifierOpt = None ) { case Seq(artifact1, artifact2) => val urls = Set( diff --git a/tests/shared/src/test/resources/resolutions/org.apache.maven/apache-maven/3.3.9 b/tests/shared/src/test/resources/resolutions/org.apache.maven/apache-maven/3.3.9 new file mode 100644 index 000000000..f6172836b --- /dev/null +++ b/tests/shared/src/test/resources/resolutions/org.apache.maven/apache-maven/3.3.9 @@ -0,0 +1,44 @@ +aopalliance:aopalliance:1.0:compile +com.google.guava:guava:18.0:compile +com.google.inject:guice:4.0:compile +commons-cli:commons-cli:1.2:compile +commons-io:commons-io:2.2:compile +commons-lang:commons-lang:2.6:compile +commons-logging:commons-logging:1.1.3:compile +javax.annotation:jsr250-api:1.0:compile +javax.enterprise:cdi-api:1.0:compile +javax.inject:javax.inject:1:compile +org.apache.commons:commons-lang3:3.4:compile +org.apache.maven:apache-maven:3.3.9:compile +org.apache.maven:maven-aether-provider:3.3.9:compile +org.apache.maven:maven-artifact:3.3.9:compile +org.apache.maven:maven-builder-support:3.3.9:compile +org.apache.maven:maven-compat:3.3.9:compile +org.apache.maven:maven-core:3.3.9:compile +org.apache.maven:maven-embedder:3.3.9:compile +org.apache.maven:maven-model:3.3.9:compile +org.apache.maven:maven-model-builder:3.3.9:compile +org.apache.maven:maven-plugin-api:3.3.9:compile +org.apache.maven:maven-repository-metadata:3.3.9:compile +org.apache.maven:maven-settings:3.3.9:compile +org.apache.maven:maven-settings-builder:3.3.9:compile +org.apache.maven.wagon:wagon-file:2.10:compile +org.apache.maven.wagon:wagon-http:2.10:compile +org.apache.maven.wagon:wagon-http-shared:2.10:compile +org.apache.maven.wagon:wagon-provider-api:2.10:compile +org.codehaus.plexus:plexus-classworlds:2.5.2:compile +org.codehaus.plexus:plexus-component-annotations:1.6:compile +org.codehaus.plexus:plexus-interpolation:1.21:compile +org.codehaus.plexus:plexus-utils:3.0.22:compile +org.eclipse.aether:aether-api:1.0.2.v20150114:compile +org.eclipse.aether:aether-connector-basic:1.0.2.v20150114:compile +org.eclipse.aether:aether-impl:1.0.2.v20150114:compile +org.eclipse.aether:aether-spi:1.0.2.v20150114:compile +org.eclipse.aether:aether-transport-wagon:1.0.2.v20150114:compile +org.eclipse.aether:aether-util:1.0.2.v20150114:compile +org.eclipse.sisu:org.eclipse.sisu.inject:0.3.2:compile +org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.2:compile +org.jsoup:jsoup:1.7.2:compile +org.slf4j:slf4j-api:1.7.5:compile +org.sonatype.plexus:plexus-cipher:1.7:compile +org.sonatype.plexus:plexus-sec-dispatcher:1.3:compile diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index 382395fdc..74edbdd48 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -121,27 +121,31 @@ object CentralTests extends TestSuite { version: String, artifactType: String, attributes: Attributes = Attributes(), - extraRepo: Option[Repository] = None + extraRepo: Option[Repository] = None, + classifierOpt: Option[String] = None, + transitive: Boolean = false )( f: Seq[Artifact] => T ): Future[T] = { - val dep = Dependency(module, version, transitive = false, attributes = attributes) - withArtifacts(dep, artifactType, extraRepo)(f) + val dep = Dependency(module, version, transitive = transitive, attributes = attributes) + withArtifacts(dep, artifactType, extraRepo, classifierOpt)(f) } def withArtifacts[T]( dep: Dependency, artifactType: String, - extraRepo: Option[Repository] + extraRepo: Option[Repository], + classifierOpt: Option[String] )( f: Seq[Artifact] => T ): Future[T] = - withArtifacts(Set(dep), artifactType, extraRepo)(f) + withArtifacts(Set(dep), artifactType, extraRepo, classifierOpt)(f) def withArtifacts[T]( deps: Set[Dependency], artifactType: String, - extraRepo: Option[Repository] + extraRepo: Option[Repository], + classifierOpt: Option[String] )( f: Seq[Artifact] => T ): Future[T] = async { @@ -151,7 +155,7 @@ object CentralTests extends TestSuite { assert(res.conflicts.isEmpty) assert(res.isDone) - val artifacts = res.dependencyArtifacts.map(_._2).filter { a => + val artifacts = classifierOpt.fold(res.dependencyArtifacts)(c => res.dependencyClassifiersArtifacts(Seq(c))).map(_._2).filter { a => a.`type` == artifactType } @@ -372,6 +376,7 @@ object CentralTests extends TestSuite { intransitiveCompiler("optional") ), "jar", + None, None ) { case Seq() => @@ -545,6 +550,35 @@ object CentralTests extends TestSuite { "0.8.0" ) } + + 'tarGzZipArtifacts - { + val mod = Module("org.apache.maven", "apache-maven") + val version = "3.3.9" + + * - resolutionCheck(mod, version) + + val expectedTarGzArtifactUrls = Set( + "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.tar.gz", + "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.tar.gz" + ) + + val expectedZipArtifactUrls = Set( + "https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip", + "https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3-bin.zip" + ) + + * - withArtifacts(mod, version, "tar.gz", classifierOpt = Some("bin"), transitive = true) { artifacts => + assert(artifacts.length == 2) + val urls = artifacts.map(_.url).toSet + assert(urls == expectedTarGzArtifactUrls) + } + + * - withArtifacts(mod, version, "zip", classifierOpt = Some("bin"), transitive = true) { artifacts => + assert(artifacts.length == 2) + val urls = artifacts.map(_.url).toSet + assert(urls == expectedZipArtifactUrls) + } + } } } From 7e32f90a9eab2b033be2f0ab7f5557bf9b013f05 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 11 Apr 2017 14:40:46 +0200 Subject: [PATCH 2/2] Rework artifact listing for Maven repositories So that signature files, checksums, etc. don't appear as standalone artifacts --- .../scala/coursier/maven/MavenSource.scala | 213 +++++++++++++----- 1 file changed, 161 insertions(+), 52 deletions(-) diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 16d5812cb..07c3e02f3 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -13,7 +13,7 @@ final case class MavenSource( import Repository._ import MavenRepository._ - def artifacts( + private def artifactsUnknownPublications( dependency: Dependency, project: Project, overrideClassifiers: Option[Seq[String]] @@ -46,7 +46,6 @@ final case class MavenSource( .withDefaultChecksums if (publication.ext == "jar") - // TODO Get available signature / checksums from directory listing artifact = artifact.withDefaultSignature artifact @@ -61,77 +60,187 @@ final case class MavenSource( ) } - val publications0 = overrideClassifiers match { + overrideClassifiers match { case Some(classifiers) => - val classifiersSet = classifiers.toSet - if (project.publications.isEmpty) - // For repositories not providing directory listings, give a try to some publications anyway - classifiers.map { classifier => - Publication( - dependency.module.name, - "jar", - "jar", - classifier - ) - } - else - project.publications.collect { - case (_, p) if classifiersSet(p.classifier) => - p - } + classifiers.map { classifier => + Publication( + dependency.module.name, + "jar", + "jar", + classifier + ) + }.map(artifactWithExtra) case None => - if (project.publications.isEmpty) { + val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type` - // For repositories not providing directory listings, give a try to some publications anyway + val extension = MavenSource.typeExtension(type0) - val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type` + val classifier = + if (dependency.attributes.classifier.isEmpty) + MavenSource.typeDefaultClassifier(type0) + else + dependency.attributes.classifier - val extension = MavenSource.typeExtension(type0) - - val classifier = - if (dependency.attributes.classifier.isEmpty) - MavenSource.typeDefaultClassifier(type0) - else - dependency.attributes.classifier - - Seq( - Publication( - dependency.module.name, - type0, - extension, - classifier - ) + Seq( + Publication( + dependency.module.name, + type0, + extension, + classifier ) - } else if (dependency.attributes.classifier.nonEmpty) + ).map(artifactWithExtra) + } + } + + private def artifactsKnownPublications( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ): Seq[Artifact] = { + + final case class EnrichedPublication( + publication: Publication, + extra: Map[String, EnrichedPublication] + ) { + def artifact: Artifact = { + + val versioning = project + .snapshotVersioning + .flatMap(versioning => + mavenVersioning(versioning, publication.classifier, publication.`type`) + ) + + val path = dependency.module.organization.split('.').toSeq ++ Seq( + MavenRepository.dirModuleName(dependency.module, sbtAttrStub), + project.actualVersion, + s"${dependency.module.name}-${versioning getOrElse project.actualVersion}${Some(publication.classifier).filter(_.nonEmpty).map("-" + _).mkString}.${publication.ext}" + ) + + val changing0 = changing.getOrElse(project.actualVersion.contains("-SNAPSHOT")) + + val extra0 = extra.mapValues(_.artifact).iterator.toMap + + Artifact( + root + path.mkString("/"), + extra0.filterKeys(MavenSource.checksumTypes).mapValues(_.url).iterator.toMap, + extra0, + publication.attributes, + changing = changing0, + authentication = authentication + ) + } + } + + def extensionIsExtra(publications: Seq[EnrichedPublication], ext: String, tpe: String): Seq[EnrichedPublication] = { + + val (withExt, other) = publications.partition(_.publication.ext.endsWith("." + ext)) + + var withExtMap = withExt.map(p => (p.publication.classifier, p.publication.ext.stripSuffix("." + ext)) -> p).toMap + + other.map { p => + val key = (p.publication.classifier, p.publication.ext) + withExtMap.get(key).fold(p) { sigPub => + withExtMap -= key + p.copy( + extra = p.extra + (tpe -> sigPub) + ) + } + } ++ withExtMap.values + } + + def groupedEnrichedPublications(publications: Seq[Publication]): Seq[EnrichedPublication] = { + + def helperSameName(publications: Seq[Publication]): Seq[EnrichedPublication] = { + + val publications0 = publications.map { pub => + EnrichedPublication(pub, Map()) + } + + Seq("sha1" -> "SHA-1", "md5" -> "MD5", "asc" -> "sig").foldLeft(publications0) { + case (pub, (ext, tpe)) => + extensionIsExtra(pub, ext, tpe) + } + } + + publications + .groupBy(_.name) + .mapValues(helperSameName) + .values + .toVector + .flatten + } + + val enrichedPublications = groupedEnrichedPublications(project.publications.map(_._2)) + + val metadataArtifactOpt = enrichedPublications.collectFirst { + case pub if pub.publication.name == dependency.module.name && + pub.publication.ext == "pom" && + pub.publication.classifier.isEmpty => + pub.artifact + } + + def withMetadataExtra(artifact: Artifact) = + metadataArtifactOpt.fold(artifact) { metadataArtifact => + artifact.copy( + extra = artifact.extra + ("metadata" -> metadataArtifact) + ) + } + + val res = overrideClassifiers match { + case Some(classifiers) => + val classifiersSet = classifiers.toSet + + enrichedPublications.collect { + case p if classifiersSet(p.publication.classifier) => + p.artifact + } + + case None => + + if (dependency.attributes.classifier.nonEmpty) // FIXME We're ignoring dependency.attributes.`type` in this case - project.publications.collect { - case (_, p) if p.classifier == dependency.attributes.classifier => - p + enrichedPublications.collect { + case p if p.publication.classifier == dependency.attributes.classifier => + p.artifact } else if (dependency.attributes.`type`.nonEmpty) - project.publications.collect { - case (_, p) - if p.`type` == dependency.attributes.`type` || - (p.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.`type`)) // wow - => - p + enrichedPublications.collect { + case p + if p.publication.`type` == dependency.attributes.`type` || + (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow + => + p.artifact } else - project.publications.collect { - case (_, p) if p.classifier.isEmpty => - p + enrichedPublications.collect { + case p if p.publication.classifier.isEmpty => + p.artifact } } - publications0.map(artifactWithExtra) + res.map(withMetadataExtra) + } + + def artifacts( + dependency: Dependency, + project: Project, + overrideClassifiers: Option[Seq[String]] + ): Seq[Artifact] = { + + if (project.publications.isEmpty) + artifactsUnknownPublications(dependency, project, overrideClassifiers) + else + artifactsKnownPublications(dependency, project, overrideClassifiers) } } object MavenSource { - + + private val checksumTypes = Set("MD5", "SHA-1") + val typeExtensions: Map[String, String] = Map( "eclipse-plugin" -> "jar", "maven-plugin" -> "jar",