From 48c58d1e96f4474b9b7e7fad6d7161db138d6737 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 19 Sep 2018 15:22:27 +0200 Subject: [PATCH] Don't rely on directory listings anymore --- .../coursier/maven/MavenRepository.scala | 103 +--------- .../scala/coursier/maven/MavenSource.scala | 192 +----------------- tests/metadata | 2 +- 3 files changed, 15 insertions(+), 282 deletions(-) diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 4268e61fb..35cedf994 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -316,110 +316,21 @@ final case class MavenRepository( proj <- Pom.project(xml, relocationAsDependency = true).right } yield proj - def isArtifact(fileName: String, prefix: String): Option[(String, String)] = - // TODO There should be a regex for that... - if (fileName.startsWith(prefix)) { - val end = fileName.stripPrefix(prefix) - val idx = end.indexOf('.') - if (idx >= 0) { - val ext = end.drop(idx + 1) - val rem = end.take(idx) - if (rem.isEmpty) - Some(("", ext)) - else if (rem.startsWith("-")) - Some((rem.drop(1), ext)) - else - None - } else - None - } else - None - val projectArtifact0 = projectArtifact(module, version, versioningValue) - val listFilesUrl = urlFor(moduleVersionPath(module, version)) + "/" - - val changing0 = changing.getOrElse(isSnapshot(version)) - - val listFilesArtifact = - Artifact( - listFilesUrl, - Map.empty, - Map( - "metadata" -> projectArtifact0 - ), - Attributes("", ""), - changing = changing0, - authentication - ) - - val requiringDirListingProjectArtifact = projectArtifact0 - .copy( - extra = projectArtifact0.extra + ( - // In LocalUpdate and LocalUpdateChanging mode, makes getting the POM fail if the POM - // is in cache, but no info about the directory listing is (directory listing not in cache and no kept error - // for it). - "required" -> listFilesArtifact - ) - ) - for { - str <- fetch(requiringDirListingProjectArtifact) - rawListFilesPageOpt <- EitherT(F.map(fetch(artifactFor(listFilesUrl, changing0)).run) { - e => Right(e.right.toOption): Either[String, Option[String]] - }) + str <- fetch(projectArtifact0) proj0 <- EitherT(F.point[Either[String, Project]](parseRawPom(str))) - } yield { - - val foundPublications = - rawListFilesPageOpt match { - case Some(rawListFilesPage) => - - val files = WebPage.listFiles(listFilesUrl, rawListFilesPage) - - val prefix = s"${module.name}-${versioningValue.getOrElse(version)}" - - val packagingTpeMap = proj0.packagingOpt - .filter(_ != Pom.relocatedPackaging) - .map { packaging => - (MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging - } - .toMap - - files - .flatMap(isArtifact(_, prefix)) - .map { - case (classifier, ext) => - val tpe = packagingTpeMap.getOrElse( - (classifier, ext), - MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext) - ) - val config = MavenSource.typeDefaultConfig(tpe).getOrElse("compile") - config -> Publication( - module.name, - tpe, - ext, - classifier - ) - } - - case None => - // Publications can't be listed - MavenSource then handles that - Nil - } - - val proj = Pom.addOptionalDependenciesInConfig( - proj0.copy(configurations = defaultConfigurations), + } yield + Pom.addOptionalDependenciesInConfig( + proj0.copy( + actualVersionOpt = Some(version), + configurations = defaultConfigurations + ), Set("", "default"), "optional" ) - - proj.copy( - actualVersionOpt = Some(version), - publications = foundPublications - ) - } } def find[F[_]]( diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 4ca7ae071..2fcccfa10 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -32,7 +32,11 @@ final case class MavenSource( val versioning = project .snapshotVersioning .flatMap(versioning => - mavenVersioning(versioning, publication.classifier, publication.`type`) + mavenVersioning( + versioning, + publication.classifier, + MavenSource.typeExtension(publication.`type`) + ) ) val path = dependency.module.organization.split('.').toSeq ++ Seq( @@ -52,6 +56,7 @@ final case class MavenSource( authentication = authentication ) .withDefaultChecksums + .withDefaultSignature if (publication.ext == "jar") artifact = artifact.withDefaultSignature @@ -133,142 +138,6 @@ final case class MavenSource( .map(artifactWithExtra) } - private val types = Map("sha1" -> "SHA-1", "sha256" -> "SHA-256", "md5" -> "MD5", "asc" -> "sig") - - private def artifactsKnownPublications( - dependency: Dependency, - project: Project, - overrideClassifiers: Option[Seq[String]] - ): Seq[Artifact] = { - - final case class EnrichedPublication( - publication: Publication, - extra: Map[String, EnrichedPublication] - ) { - def artifact: Artifact = - artifact(publication.`type`) - def artifact(versioningType: String): Artifact = { - - val versioningExtension = MavenSource.typeExtensions.getOrElse(versioningType, versioningType) - - val versioning = project - .snapshotVersioning - .flatMap(versioning => - mavenVersioning(versioning, publication.classifier, versioningExtension) - ) - - val path = dependency.module.organization.split('.').toSeq ++ Seq( - MavenRepository.dirModuleName(dependency.module, sbtAttrStub), - project.actualVersion, - s"${dependency.module.name}-${versioning getOrElse project.actualVersion}${Some(publication.classifier).filter(_.nonEmpty).map("-" + _).mkString}.${publication.ext}" - ) - - val changing0 = changing.getOrElse(isSnapshot(project.actualVersion)) - - val extra0 = extra.mapValues(_.artifact(versioningType)).iterator.toMap - - Artifact( - root + path.mkString("/"), - extra0.filterKeys(MavenSource.checksumTypes).mapValues(_.url).iterator.toMap, - extra0, - publication.attributes, - changing = changing0, - authentication = authentication - ) - } - } - - def groupedEnrichedPublications(publications: Seq[Publication]): Seq[EnrichedPublication] = { - - def helper(publications: Seq[Publication]): Seq[EnrichedPublication] = { - - var publications0 = publications - .map { pub => - pub.ext -> EnrichedPublication(pub, Map()) - } - .toMap - - val byLength = publications0.toVector.sortBy(-_._1.length) - - for { - (ext, _) <- byLength - idx = ext.lastIndexOf('.') - if idx >= 0 - subExt = ext.substring(idx + 1) - baseExt = ext.substring(0, idx) - tpe <- types.get(subExt) - mainPub <- publications0.get(baseExt) - } { - val pub = publications0(ext) - publications0 += baseExt -> mainPub.copy( - extra = mainPub.extra + (tpe -> pub) - ) - publications0 -= ext - } - - publications0.values.toVector - } - - publications - .groupBy(p => (p.name, p.classifier)) - .mapValues(helper) - .values - .toVector - .flatten - } - - val enrichedPublications = groupedEnrichedPublications(project.publications.map(_._2)) - - val metadataArtifactOpt = enrichedPublications.collectFirst { - case pub if pub.publication.name == dependency.module.name && - pub.publication.ext == "pom" && - pub.publication.classifier.isEmpty => - pub.artifact - } - - def withMetadataExtra(artifact: Artifact) = - metadataArtifactOpt.fold(artifact) { metadataArtifact => - artifact.copy( - extra = artifact.extra + ("metadata" -> metadataArtifact) - ) - } - - val res = overrideClassifiers match { - case Some(classifiers) => - val classifiersSet = classifiers.toSet - - enrichedPublications.collect { - case p if classifiersSet(p.publication.classifier) => - p.artifact - } - - case None => - - if (dependency.attributes.classifier.nonEmpty) - // FIXME We're ignoring dependency.attributes.`type` in this case - enrichedPublications.collect { - case p if p.publication.classifier == dependency.attributes.classifier => - p.artifact - } - else if (dependency.attributes.`type`.nonEmpty) - enrichedPublications.collect { - case p - if (p.publication.classifier.isEmpty || p.publication.classifier == MavenSource.typeDefaultClassifier(dependency.attributes.`type`)) && ( - p.publication.`type` == dependency.attributes.`type` || - (p.publication.ext == dependency.attributes.`type` && project.packagingOpt.toSeq.contains(p.publication.`type`)) // wow - ) => - p.artifact - } - else - enrichedPublications.collect { - case p if p.publication.classifier.isEmpty => - p.artifact - } - } - - res.map(withMetadataExtra) - } - private val dummyArtifact = Artifact("", Map(), Map(), Attributes("", ""), changing = false, None) def artifacts( @@ -285,52 +154,13 @@ final case class MavenSource( extra = a.extra.mapValues(makeOptional).iterator.toMap + (Artifact.optionalKey -> dummyArtifact) ) - def merge(a: Artifact, other: Artifact): Artifact = { - - assert(a.url == other.url, s"Merging artifacts with different URLs (${a.url}, ${other.url})") - - val extra = - a.extra.map { - case (k, v) => - k -> other.extra.get(k).fold(v)(merge(v, _)) - } ++ - other.extra - .filterKeys(k => !a.extra.contains(k) && k != Artifact.optionalKey) - - a.copy( - checksumUrls = other.checksumUrls ++ a.checksumUrls, - extra = extra - ) - } - - val defaultPublications = artifactsUnknownPublications(dependency, project, overrideClassifiers) + artifactsUnknownPublications(dependency, project, overrideClassifiers) .map(makeOptional) - - if (project.publications.isEmpty) - defaultPublications - else { - val listedPublications = artifactsKnownPublications(dependency, project, overrideClassifiers) - val listedUrls = listedPublications.map(_.url).toSet - val defaultPublicationsMap = defaultPublications - .map(a => a.url -> a) - .toMap - val listedPublications0 = listedPublications.map { a => - defaultPublicationsMap - .get(a.url) - .fold(a)(merge(a, _)) - } - val extraPublications = defaultPublications - .filter(a => !listedUrls(a.url)) - - listedPublications0 ++ extraPublications - } } } object MavenSource { - private val checksumTypes = Set("MD5", "SHA-1", "SHA-256") - val typeExtensions: Map[String, String] = Map( "eclipse-plugin" -> "jar", "maven-plugin" -> "jar", @@ -373,12 +203,4 @@ object MavenSource { def classifierExtensionDefaultTypeOpt(classifier: String, ext: String): Option[String] = classifierExtensionDefaultTypes.get((classifier, ext)) - val typeDefaultConfigs: Map[String, String] = Map( - "doc" -> "docs", - "src" -> "sources" - ) - - def typeDefaultConfig(`type`: String): Option[String] = - typeDefaultConfigs.get(`type`) - } diff --git a/tests/metadata b/tests/metadata index ed446075a..cdad75ba7 160000 --- a/tests/metadata +++ b/tests/metadata @@ -1 +1 @@ -Subproject commit ed446075ace7914af71a39ac7b00f17fa5d9190b +Subproject commit cdad75ba739422d2f110e0e70093e21c4cb0e6d6