From ed25bcba4335acd4d1e53c650a75a0761ecd09a2 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 14 Nov 2017 10:57:29 +0100 Subject: [PATCH] Add implementation of coursier --- build.sbt | 19 +- .../LibraryManagementInterface.scala | 2 +- .../src/main/scala/coursier/FromSbt.scala | 283 ++++++++++++++++++ coursier/src/main/scala/coursier/ToSbt.scala | 275 +++++++++++++++++ .../CoursierDependencyResolution.scala | 243 +++++++++++++++ .../coursier/Resolvers.scala | 50 ++++ .../coursier/BaseCoursierSpecification.scala | 37 +++ .../coursier/ResolutionSpec.scala | 142 +++++++++ .../librarymanagement/coursier/UnitSpec.scala | 5 + project/Dependencies.scala | 11 +- 10 files changed, 1062 insertions(+), 5 deletions(-) create mode 100644 coursier/src/main/scala/coursier/FromSbt.scala create mode 100644 coursier/src/main/scala/coursier/ToSbt.scala create mode 100644 coursier/src/main/scala/sbt/librarymanagement/coursier/CoursierDependencyResolution.scala create mode 100644 coursier/src/main/scala/sbt/librarymanagement/coursier/Resolvers.scala create mode 100644 coursier/src/test/scala/sbt/librarymanagement/coursier/BaseCoursierSpecification.scala create mode 100644 coursier/src/test/scala/sbt/librarymanagement/coursier/ResolutionSpec.scala create mode 100644 coursier/src/test/scala/sbt/librarymanagement/coursier/UnitSpec.scala diff --git a/build.sbt b/build.sbt index 0c50280ab..3a4863d9d 100644 --- a/build.sbt +++ b/build.sbt @@ -12,6 +12,7 @@ def commonSettings: Seq[Setting[_]] = Def.settings( // publishArtifact in packageDoc := false, resolvers += Resolver.typesafeIvyRepo("releases"), resolvers += Resolver.sonatypeRepo("snapshots"), + resolvers += Resolver.sbtPluginRepo("releases"), resolvers += "bintray-sbt-maven-releases" at "https://dl.bintray.com/sbt/maven-releases/", // concurrentRestrictions in Global += Util.testExclusiveRestriction, testOptions += Tests.Argument(TestFrameworks.ScalaCheck, "-w", "1"), @@ -45,7 +46,7 @@ val mimaSettings = Def settings ( ) lazy val lmRoot = (project in file(".")) - .aggregate(lmCore, lmIvy) + .aggregate(lmCore, lmIvy, lmCoursier) .settings( inThisBuild( Seq( @@ -259,6 +260,22 @@ lazy val lmIvy = (project in file("ivy")) ), ) +lazy val lmCoursier = (project in file("coursier")) + .enablePlugins(ContrabandPlugin, JsonCodecPlugin) + .dependsOn(lmCore) + .settings( + commonSettings, + crossScalaVersions := Seq(scala212, scala211), + name := "librarymanagement-coursier", + libraryDependencies ++= Seq(coursier, coursierCache, scalaTest, scalaCheck), + managedSourceDirectories in Compile += + baseDirectory.value / "src" / "main" / "contraband-scala", + sourceManaged in (Compile, generateContrabands) := baseDirectory.value / "src" / "main" / "contraband-scala", + contrabandFormatsForType in generateContrabands in Compile := DatatypeConfig.getFormats, + scalacOptions in (Compile, console) --= + Vector("-Ywarn-unused-import", "-Ywarn-unused", "-Xlint") + ) + lazy val lmScriptedTest = (project in file("scripted-test")) .enablePlugins(SbtPlugin) .settings( diff --git a/core/src/main/scala/sbt/librarymanagement/LibraryManagementInterface.scala b/core/src/main/scala/sbt/librarymanagement/LibraryManagementInterface.scala index 33c72c55f..5236a1a2a 100644 --- a/core/src/main/scala/sbt/librarymanagement/LibraryManagementInterface.scala +++ b/core/src/main/scala/sbt/librarymanagement/LibraryManagementInterface.scala @@ -66,7 +66,7 @@ trait PublisherInterface { } /** - * Decribes the representation of a module, inclding its dependencies + * Decribes the representation of a module, including its dependencies * and the version of Scala it uses, if any. */ trait ModuleDescriptor { diff --git a/coursier/src/main/scala/coursier/FromSbt.scala b/coursier/src/main/scala/coursier/FromSbt.scala new file mode 100644 index 000000000..6539e1d97 --- /dev/null +++ b/coursier/src/main/scala/coursier/FromSbt.scala @@ -0,0 +1,283 @@ +package coursier + +import coursier.ivy.IvyRepository +import coursier.ivy.IvyXml.{ mappings => ivyXmlMappings } +import java.net.{ MalformedURLException, URL } + +import coursier.core.Authentication +import sbt.internal.librarymanagement.mavenint.SbtPomExtraProperties +import sbt.librarymanagement._ +import sbt.librarymanagement.Resolver +import sbt.util.Logger + +object FromSbt { + + def sbtModuleIdName( + moduleId: ModuleID, + scalaVersion: => String, + scalaBinaryVersion: => String + ): String = + sbtCrossVersionName(moduleId.name, moduleId.crossVersion, scalaVersion, scalaBinaryVersion) + + def sbtCrossVersionName( + name: String, + crossVersion: CrossVersion, + scalaVersion: => String, + scalaBinaryVersion: => String + ): String = + CrossVersion(crossVersion, scalaVersion, scalaBinaryVersion) + .fold(name)(_(name)) + + def attributes(attr: Map[String, String]): Map[String, String] = + attr + .map { + case (k, v) => + k.stripPrefix("e:") -> v + } + .filter { + case (k, _) => + !k.startsWith(SbtPomExtraProperties.POM_INFO_KEY_PREFIX) + } + + def moduleVersion( + module: ModuleID, + scalaVersion: String, + scalaBinaryVersion: String + ): (Module, String) = { + + val fullName = sbtModuleIdName(module, scalaVersion, scalaBinaryVersion) + + val module0 = + Module(module.organization, fullName, FromSbt.attributes(module.extraDependencyAttributes)) + val version = module.revision + + (module0, version) + } + + def dependencies( + module: ModuleID, + scalaVersion: String, + scalaBinaryVersion: String + ): Seq[(String, Dependency)] = { + + // TODO Warn about unsupported properties in `module` + + val (module0, version) = moduleVersion(module, scalaVersion, scalaBinaryVersion) + + val dep = Dependency( + module0, + version, + exclusions = module.exclusions.map { rule => + // FIXME Other `rule` fields are ignored here + (rule.organization, rule.name) + }.toSet, + transitive = module.isTransitive + ) + + val mapping = module.configurations.getOrElse("compile") + val allMappings = ivyXmlMappings(mapping) + + val attributes = + if (module.explicitArtifacts.isEmpty) + Seq(Attributes("", "")) + else + module.explicitArtifacts.map { a => + Attributes(`type` = a.`type`, classifier = a.classifier.getOrElse("")) + } + + for { + (from, to) <- allMappings + attr <- attributes + } yield from -> dep.copy(configuration = to, attributes = attr) + } + + def fallbackDependencies( + allDependencies: Seq[ModuleID], + scalaVersion: String, + scalaBinaryVersion: String + ): Seq[(Module, String, URL, Boolean)] = + for { + module <- allDependencies + artifact <- module.explicitArtifacts + url <- artifact.url.toSeq + } yield { + val (module0, version) = moduleVersion(module, scalaVersion, scalaBinaryVersion) + (module0, version, url, module.isChanging) + } + + def sbtClassifiersProject( + cm: GetClassifiersModule, + scalaVersion: String, + scalaBinaryVersion: String + ) = { + + val p = FromSbt.project( + cm.id, + cm.dependencies, + cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap, + scalaVersion, + scalaBinaryVersion + ) + + // for w/e reasons, the dependencies sometimes don't land in the right config above + // this is a loose attempt at fixing that + cm.configurations match { + case Seq(cfg) => + p.copy( + dependencies = p.dependencies.map { + case (_, d) => (cfg.name, d) + } + ) + case _ => + p + } + } + + def project( + projectID: ModuleID, + allDependencies: Seq[ModuleID], + ivyConfigurations: Map[String, Seq[String]], + scalaVersion: String, + scalaBinaryVersion: String + ): Project = { + + val deps = allDependencies.flatMap(dependencies(_, scalaVersion, scalaBinaryVersion)) + + Project( + Module( + projectID.organization, + sbtModuleIdName(projectID, scalaVersion, scalaBinaryVersion), + FromSbt.attributes(projectID.extraDependencyAttributes) + ), + projectID.revision, + deps, + ivyConfigurations, + None, + Nil, + Nil, + Nil, + None, + None, + None, + None, + Nil, + Info.empty + ) + } + + private def mavenCompatibleBaseOpt(patterns: Patterns): Option[String] = + if (patterns.isMavenCompatible) { + val baseIvyPattern = patterns.ivyPatterns.head.takeWhile(c => c != '[' && c != '(') + val baseArtifactPattern = patterns.ivyPatterns.head.takeWhile(c => c != '[' && c != '(') + + if (baseIvyPattern == baseArtifactPattern) + Some(baseIvyPattern) + else + None + } else + None + + private def mavenRepositoryOpt( + root: String, + log: Logger, + authentication: Option[Authentication] + ): Option[MavenRepository] = + try { + Cache.url(root) // ensure root is a URL whose protocol can be handled here + val root0 = if (root.endsWith("/")) root else root + "/" + Some( + MavenRepository( + root0, + authentication = authentication + ) + ) + } catch { + case e: MalformedURLException => + log.warn( + "Error parsing Maven repository base " + + root + + Option(e.getMessage).fold("")(" (" + _ + ")") + + ", ignoring it" + ) + + None + } + + def repository( + resolver: Resolver, + ivyProperties: Map[String, String], + log: Logger, + authentication: Option[Authentication] + ): Option[Repository] = + resolver match { + case r: sbt.librarymanagement.MavenRepository => + mavenRepositoryOpt(r.root, log, authentication) + + case r: FileRepository + if r.patterns.ivyPatterns.lengthCompare(1) == 0 && + r.patterns.artifactPatterns.lengthCompare(1) == 0 => + val mavenCompatibleBaseOpt0 = mavenCompatibleBaseOpt(r.patterns) + + mavenCompatibleBaseOpt0 match { + case None => + val repo = IvyRepository.parse( + "file://" + r.patterns.artifactPatterns.head, + metadataPatternOpt = Some("file://" + r.patterns.ivyPatterns.head), + changing = Some(true), + properties = ivyProperties, + dropInfoAttributes = true, + authentication = authentication + ) match { + case Left(err) => + sys.error( + s"Cannot parse Ivy patterns ${r.patterns.artifactPatterns.head} and ${r.patterns.ivyPatterns.head}: $err" + ) + case Right(repo) => + repo + } + + Some(repo) + + case Some(mavenCompatibleBase) => + mavenRepositoryOpt("file://" + mavenCompatibleBase, log, authentication) + } + + case r: URLRepository + if r.patterns.ivyPatterns.lengthCompare(1) == 0 && + r.patterns.artifactPatterns.lengthCompare(1) == 0 => + val mavenCompatibleBaseOpt0 = mavenCompatibleBaseOpt(r.patterns) + + mavenCompatibleBaseOpt0 match { + case None => + val repo = IvyRepository.parse( + r.patterns.artifactPatterns.head, + metadataPatternOpt = Some(r.patterns.ivyPatterns.head), + changing = None, + properties = ivyProperties, + dropInfoAttributes = true, + authentication = authentication + ) match { + case Left(err) => + sys.error( + s"Cannot parse Ivy patterns ${r.patterns.artifactPatterns.head} and ${r.patterns.ivyPatterns.head}: $err" + ) + case Right(repo) => + repo + } + + Some(repo) + + case Some(mavenCompatibleBase) => + mavenRepositoryOpt(mavenCompatibleBase, log, authentication) + } + + case raw: RawRepository + if raw.name == "inter-project" => // sbt.RawRepository.equals just compares names anyway + None + + case other => + log.warn(s"Unrecognized repository ${other.name}, ignoring it") + None + } + +} diff --git a/coursier/src/main/scala/coursier/ToSbt.scala b/coursier/src/main/scala/coursier/ToSbt.scala new file mode 100644 index 000000000..ec979dab6 --- /dev/null +++ b/coursier/src/main/scala/coursier/ToSbt.scala @@ -0,0 +1,275 @@ +package coursier + +import java.io.File +import java.net.URL +import java.util.GregorianCalendar +import java.util.concurrent.ConcurrentHashMap + +import coursier.maven.MavenSource +import sbt.librarymanagement._ +import sbt.util.Logger + +object ToSbt { + + private def caching[K, V](f: K => V): K => V = { + + val cache = new ConcurrentHashMap[K, V] + + key => + val previousValueOpt = Option(cache.get(key)) + + previousValueOpt.getOrElse { + val value = f(key) + val concurrentValueOpt = Option(cache.putIfAbsent(key, value)) + concurrentValueOpt.getOrElse(value) + } + } + + val moduleId = caching[(Dependency, Map[String, String]), ModuleID] { + case (dependency, extraProperties) => + sbt.librarymanagement + .ModuleID( + dependency.module.organization, + dependency.module.name, + dependency.version + ) + .withConfigurations( + Some(dependency.configuration) + ) + .withExtraAttributes( + dependency.module.attributes ++ extraProperties + ) + .withExclusions( + dependency.exclusions.toVector + .map { + case (org, name) => + sbt.librarymanagement + .InclExclRule() + .withOrganization(org) + .withName(name) + } + ) + .withIsTransitive( + dependency.transitive + ) + } + + val artifact = caching[(Module, Map[String, String], Artifact), sbt.librarymanagement.Artifact] { + case (module, extraProperties, artifact) => + sbt.librarymanagement + .Artifact(module.name) + // FIXME Get these two from publications + .withType(artifact.attributes.`type`) + .withExtension(MavenSource.typeExtension(artifact.attributes.`type`)) + .withClassifier( + Some(artifact.attributes.classifier) + .filter(_.nonEmpty) + .orElse(MavenSource.typeDefaultClassifierOpt(artifact.attributes.`type`)) + ) + // .withConfigurations(Vector()) + .withUrl(Some(new URL(artifact.url))) + .withExtraAttributes(module.attributes ++ extraProperties) + } + + val moduleReport = + caching[(Dependency, Seq[(Dependency, Project)], Project, Seq[(Artifact, Option[File])]), + ModuleReport] { + case (dependency, dependees, project, artifacts) => + val sbtArtifacts = artifacts.collect { + case (artifact, Some(file)) => + (ToSbt.artifact((dependency.module, project.properties.toMap, artifact)), file) + } + val sbtMissingArtifacts = artifacts.collect { + case (artifact, None) => + ToSbt.artifact((dependency.module, project.properties.toMap, artifact)) + } + + val publicationDate = project.info.publication.map { dt => + new GregorianCalendar(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) + } + + val callers = dependees.map { + case (dependee, dependeeProj) => + Caller( + ToSbt.moduleId((dependee, dependeeProj.properties.toMap)), + dependeeProj.configurations.keys.toVector.map(ConfigRef(_)), + dependee.module.attributes ++ dependeeProj.properties, + // FIXME Set better values here + isForceDependency = false, + isChangingDependency = false, + isTransitiveDependency = false, + isDirectlyForceDependency = false + ) + } + + ModuleReport( + ToSbt.moduleId((dependency, project.properties.toMap)), + sbtArtifacts.toVector, + sbtMissingArtifacts.toVector + ) + // .withStatus(None) + .withPublicationDate(publicationDate) + // .withResolver(None) + // .withArtifactResolver(None) + // .withEvicted(false) + // .withEvictedData(None) + // .withEvictedReason(None) + // .withProblem(None) + .withHomepage(Some(project.info.homePage).filter(_.nonEmpty)) + .withExtraAttributes(dependency.module.attributes ++ project.properties) + // .withIsDefault(None) + // .withBranch(None) + .withConfigurations(project.configurations.keys.toVector.map(ConfigRef(_))) + .withLicenses(project.info.licenses.toVector) + .withCallers(callers.toVector) + } + + private def grouped[K, V](map: Seq[(K, V)]): Map[K, Seq[V]] = + map.groupBy { case (k, _) => k }.map { + case (k, l) => + k -> l.map { case (_, v) => v } + } + + def moduleReports( + res: Resolution, + classifiersOpt: Option[Seq[String]], + artifactFileOpt: (Module, String, Artifact) => Option[File], + log: Logger, + keepPomArtifact: Boolean = false, + includeSignatures: Boolean = false + ) = { + val depArtifacts1 = + classifiersOpt match { + case None => res.dependencyArtifacts(withOptional = true) + case Some(cl) => res.dependencyClassifiersArtifacts(cl) + } + + val depArtifacts0 = + if (keepPomArtifact) + depArtifacts1 + else + depArtifacts1.filter { + case (_, a) => a.attributes != Attributes("pom", "") + } + + val depArtifacts = + if (includeSignatures) { + + val notFound = depArtifacts0.filter(!_._2.extra.contains("sig")) + + if (notFound.isEmpty) + depArtifacts0.flatMap { + case (dep, a) => + Seq(dep -> a) ++ a.extra.get("sig").toSeq.map(dep -> _) + } else { + for ((_, a) <- notFound) + log.error(s"No signature found for ${a.url}") + sys.error(s"${notFound.length} signature(s) not found") + } + } else + depArtifacts0 + + val groupedDepArtifacts = grouped(depArtifacts) + + val versions = res.dependencies.toVector.map { dep => + dep.module -> dep.version + }.toMap + + def clean(dep: Dependency): Dependency = + dep.copy(configuration = "", exclusions = Set.empty, optional = false) + + val reverseDependencies = res.reverseDependencies.toVector + .map { + case (k, v) => + clean(k) -> v.map(clean) + } + .groupBy { case (k, v) => k } + .mapValues { v => + v.flatMap { + case (_, l) => l + } + } + .toVector + .toMap + + groupedDepArtifacts.map { + case (dep, artifacts) => + val (_, proj) = res.projectCache(dep.moduleVersion) + + // FIXME Likely flaky... + val dependees = reverseDependencies + .getOrElse(clean(dep.copy(version = "")), Vector.empty) + .map { dependee0 => + val version = versions(dependee0.module) + val dependee = dependee0.copy(version = version) + val (_, dependeeProj) = res.projectCache(dependee.moduleVersion) + (dependee, dependeeProj) + } + + ToSbt.moduleReport( + ( + dep, + dependees, + proj, + artifacts.map(a => a -> artifactFileOpt(proj.module, proj.version, a)) + )) + } + } + + def updateReport( + configDependencies: Map[String, Seq[Dependency]], + resolutions: Map[String, Resolution], + configs: Map[String, Set[String]], + classifiersOpt: Option[Seq[String]], + artifactFileOpt: (Module, String, Artifact) => Option[File], + log: Logger, + keepPomArtifact: Boolean = false, + includeSignatures: Boolean = false + ): UpdateReport = { + + val configReports = configs.map { + case (config, extends0) => + val configDeps = extends0.flatMap(configDependencies.getOrElse(_, Nil)) + val subRes = resolutions(config).subset(configDeps) + + val reports = ToSbt.moduleReports( + subRes, + classifiersOpt, + artifactFileOpt, + log, + keepPomArtifact = keepPomArtifact, + includeSignatures = includeSignatures + ) + + val reports0 = + if (subRes.rootDependencies.size == 1) { + // quick hack ensuring the module for the only root dependency + // appears first in the update report, see https://github.com/coursier/coursier/issues/650 + val dep = subRes.rootDependencies.head + val (_, proj) = subRes.projectCache(dep.moduleVersion) + val mod = ToSbt.moduleId((dep, proj.properties.toMap)) + val (main, other) = reports.partition { r => + r.module.organization == mod.organization && + r.module.name == mod.name && + r.module.crossVersion == mod.crossVersion + } + main.toVector ++ other.toVector + } else + reports.toVector + + ConfigurationReport( + ConfigRef(config), + reports0, + Vector() + ) + } + + UpdateReport( + File.createTempFile("fake-update-report", "json"), + configReports.toVector, + UpdateStats(-1L, -1L, -1L, cached = false), + Map.empty + ) + } + +} diff --git a/coursier/src/main/scala/sbt/librarymanagement/coursier/CoursierDependencyResolution.scala b/coursier/src/main/scala/sbt/librarymanagement/coursier/CoursierDependencyResolution.scala new file mode 100644 index 000000000..0c409390b --- /dev/null +++ b/coursier/src/main/scala/sbt/librarymanagement/coursier/CoursierDependencyResolution.scala @@ -0,0 +1,243 @@ +package sbt.librarymanagement.coursier + +import java.io.{ File, OutputStreamWriter } + +import coursier.{ Artifact, Resolution, _ } +import coursier.util.{ Gather, Task } +import sbt.librarymanagement.Configurations.{ CompilerPlugin, Component, ScalaTool } +import sbt.librarymanagement._ +import sbt.util.Logger + +case class CoursierModuleDescriptor( + directDependencies: Vector[ModuleID], + scalaModuleInfo: Option[ScalaModuleInfo], + moduleSettings: ModuleSettings, + extraInputHash: Long +) extends ModuleDescriptor + +case class CoursierModuleSettings() extends ModuleSettings + +private[sbt] class CoursierDependencyResolution(resolvers: Seq[Resolver]) + extends DependencyResolutionInterface { + + private[coursier] val reorderedResolvers = Resolvers.reorder(resolvers) + + /** + * Builds a ModuleDescriptor that describes a subproject with dependencies. + * + * @param moduleSetting It contains the information about the module including the dependencies. + * @return A `ModuleDescriptor` describing a subproject and its dependencies. + */ + override def moduleDescriptor( + moduleSetting: ModuleDescriptorConfiguration): CoursierModuleDescriptor = { + CoursierModuleDescriptor( + moduleSetting.dependencies, + moduleSetting.scalaModuleInfo, + CoursierModuleSettings(), + 1L // FIXME: use correct value + ) + } + + /** + * Resolves the given module's dependencies performing a retrieval. + * + * @param module The module to be resolved. + * @param configuration The update configuration. + * @param uwconfig The configuration to handle unresolved warnings. + * @param log The logger. + * @return The result, either an unresolved warning or an update report. Note that this + * update report will or will not be successful depending on the `missingOk` option. + */ + override def update(module: ModuleDescriptor, + configuration: UpdateConfiguration, + uwconfig: UnresolvedWarningConfiguration, + log: Logger): Either[UnresolvedWarning, UpdateReport] = { + + if (reorderedResolvers.isEmpty) { + log.error( + "Dependency resolution is configured with an empty list of resolvers. This is unlikely to work.") + } + + val dependencies = module.directDependencies.map(toCoursierDependency).toSet + val start = Resolution(dependencies) + val authentication = None // TODO: get correct value + val ivyConfiguration = Map("ivy.home" -> "~/.ivy2/") // TODO: get correct value + val repositories = + reorderedResolvers.flatMap(r => FromSbt.repository(r, ivyConfiguration, log, authentication)) ++ Seq( + Cache.ivy2Local, + Cache.ivy2Cache) + + import scala.concurrent.ExecutionContext.Implicits.global + + val fetch = Fetch.from(repositories, Cache.fetch[Task](logger = Some(createLogger()))) + val resolution = start.process.run(fetch).unsafeRun() + + if (resolution.errors.isEmpty) { + val localArtifacts: Map[Artifact, Either[FileError, File]] = Gather[Task] + .gather( + resolution.artifacts.map { a => + Cache.file[Task](a).run.map((a, _)) + } + ) + .unsafeRun() + .toMap + toUpdateReport(resolution, localArtifacts, log) + } else { + toSbtError(log, uwconfig, resolution) + } + } + + // utilities + + private def createLogger() = { + val t = new TermDisplay(new OutputStreamWriter(System.out)) + t.init() + t + } + + private def toCoursierDependency(moduleID: ModuleID): Dependency = { + val attrs = moduleID.explicitArtifacts + .map(a => Attributes(`type` = a.`type`, classifier = a.classifier.getOrElse(""))) + .headOption + .getOrElse(Attributes()) + + // for some reason, sbt adds the prefix "e:" to extraAttributes + val extraAttrs = moduleID.extraAttributes.map { + case (key, value) => (key.replaceFirst("^e:", ""), value) + } + + Dependency( + Module(moduleID.organization, moduleID.name, extraAttrs), + moduleID.revision, + moduleID.configurations.getOrElse(""), + attrs, + exclusions = moduleID.exclusions.map { rule => + (rule.organization, rule.name) + }.toSet, + transitive = moduleID.isTransitive + ) + } + + private def toUpdateReport(resolution: Resolution, + artifactFilesOrErrors0: Map[Artifact, Either[FileError, File]], + log: Logger): Either[UnresolvedWarning, UpdateReport] = { + + val artifactFiles = artifactFilesOrErrors0.collect { + case (artifact, Right(file)) => + artifact -> file + } + + val artifactErrors = artifactFilesOrErrors0.toVector + .collect { + case (a, Left(err)) if !a.isOptional || !err.notFound => + a -> err + } + + if (artifactErrors.nonEmpty) { + // TODO: handle error the correct sbt way + throw new RuntimeException(s"Could not download dependencies: $artifactErrors") + } + + // can be non empty only if ignoreArtifactErrors is true or some optional artifacts are not found + val erroredArtifacts = artifactFilesOrErrors0.collect { + case (a, Left(_)) => a + }.toSet + + val depsByConfig = resolution.dependencies.groupBy(_.configuration).mapValues(_.toSeq) + + val configurations = extractConfigurationTree + + val configResolutions = + (depsByConfig.keys ++ configurations.keys).map(k => (k, resolution)).toMap + + val sbtBootJarOverrides = Map.empty[(Module, String), File] // TODO: get correct values + val classifiers = None // TODO: get correct values + + if (artifactErrors.isEmpty) { + Right( + ToSbt.updateReport( + depsByConfig, + configResolutions, + configurations, + classifiers, + artifactFileOpt( + sbtBootJarOverrides, + artifactFiles, + erroredArtifacts, + log, + _, + _, + _ + ), + log + )) + } else { + throw new RuntimeException(s"Could not save downloaded dependencies: $erroredArtifacts") + } + + } + + type ConfigurationName = String + type ConfigurationDependencyTree = Map[ConfigurationName, Set[ConfigurationName]] + + // Key is the name of the configuration (i.e. `compile`) and the values are the name itself plus the + // names of the configurations that this one depends on. + private def extractConfigurationTree: ConfigurationDependencyTree = { + (Configurations.default ++ Configurations.defaultInternal ++ Seq(ScalaTool, + CompilerPlugin, + Component)) + .map(c => (c.name, c.extendsConfigs.map(_.name) :+ c.name)) + .toMap + .mapValues(_.toSet) + } + + private def artifactFileOpt( + sbtBootJarOverrides: Map[(Module, String), File], + artifactFiles: Map[Artifact, File], + erroredArtifacts: Set[Artifact], + log: Logger, + module: Module, + version: String, + artifact: Artifact + ) = { + + val artifact0 = artifact + .copy(attributes = Attributes()) // temporary hack :-( + + // Under some conditions, SBT puts the scala JARs of its own classpath + // in the application classpath. Ensuring we return SBT's jars rather than + // JARs from the coursier cache, so that a same JAR doesn't land twice in the + // application classpath (once via SBT jars, once via coursier cache). + val fromBootJars = + if (artifact.classifier.isEmpty && artifact.`type` == "jar") + sbtBootJarOverrides.get((module, version)) + else + None + + val res = fromBootJars.orElse(artifactFiles.get(artifact0)) + + if (res.isEmpty && !erroredArtifacts(artifact0)) + log.error(s"${artifact.url} not downloaded (should not happen)") + + res + } + + private def toSbtError(log: Logger, + uwconfig: UnresolvedWarningConfiguration, + resolution: Resolution) = { + val failedResolution = resolution.errors.map { + case ((failedModule, failedVersion), _) => + ModuleID(failedModule.organization, failedModule.name, failedVersion) + } + val msgs = resolution.errors.flatMap(_._2) + log.debug(s"Failed resolution: $msgs") + log.debug(s"Missing artifacts: $failedResolution") + val ex = new ResolveException(msgs, failedResolution) + Left(UnresolvedWarning(ex, uwconfig)) + } +} + +object CoursierDependencyResolution { + def apply(resolvers: Seq[Resolver]) = + DependencyResolution(new CoursierDependencyResolution(resolvers)) +} diff --git a/coursier/src/main/scala/sbt/librarymanagement/coursier/Resolvers.scala b/coursier/src/main/scala/sbt/librarymanagement/coursier/Resolvers.scala new file mode 100644 index 000000000..c8513f42e --- /dev/null +++ b/coursier/src/main/scala/sbt/librarymanagement/coursier/Resolvers.scala @@ -0,0 +1,50 @@ +package sbt.librarymanagement.coursier + +import sbt.librarymanagement.{ MavenRepository, Resolver, URLRepository } + +object Resolvers { + + private val slowReposBase = Seq( + "https://repo.typesafe.com/", + "https://repo.scala-sbt.org/", + "http://repo.typesafe.com/", + "http://repo.scala-sbt.org/" + ) + + private val fastReposBase = Seq( + "http://repo1.maven.org/", + "https://repo1.maven.org/" + ) + + private def url(res: Resolver): Option[String] = + res match { + case m: MavenRepository => + Some(m.root) + case u: URLRepository => + u.patterns.artifactPatterns.headOption + .orElse(u.patterns.ivyPatterns.headOption) + case _ => + None + } + + private def filterResolvers(bases: Seq[String], + resolvers: Seq[(Resolver, Option[String])]): Seq[Resolver] = + resolvers + .filter(tuple => tuple._2.exists(url => bases.exists(base => url.startsWith(base)))) + .map(_._1) + + def reorder(resolvers: Seq[Resolver]): Seq[Resolver] = { + + val byUrl = resolvers.map(r => (r, url(r))) + + val fast = filterResolvers(fastReposBase, byUrl) + val slow = filterResolvers(slowReposBase, byUrl) + val rest = resolvers.diff(fast).diff(slow) + + val reordered = fast ++ rest ++ slow + assert(reordered.size == resolvers.size, + "Reordered resolvers should be the same size as the unordered ones.") + + reordered + } +} diff --git a/coursier/src/test/scala/sbt/librarymanagement/coursier/BaseCoursierSpecification.scala b/coursier/src/test/scala/sbt/librarymanagement/coursier/BaseCoursierSpecification.scala new file mode 100644 index 000000000..4fe2e3d0e --- /dev/null +++ b/coursier/src/test/scala/sbt/librarymanagement/coursier/BaseCoursierSpecification.scala @@ -0,0 +1,37 @@ +package sbt.librarymanagement.coursier + +import sbt.internal.librarymanagement.cross.CrossVersionUtil +import sbt.internal.util.ConsoleLogger +import sbt.librarymanagement.Configurations._ +import sbt.librarymanagement._ + +trait BaseCoursierSpecification extends UnitSpec { + lazy val log = ConsoleLogger() + val lmEngine: CoursierDependencyResolution + + def configurations = Vector(Compile, Test, Runtime) + def module(moduleId: ModuleID, + deps: Vector[ModuleID], + scalaFullVersion: Option[String], + overrideScalaVersion: Boolean = true): ModuleDescriptor = { + val scalaModuleInfo = scalaFullVersion map { fv => + ScalaModuleInfo( + scalaFullVersion = fv, + scalaBinaryVersion = CrossVersionUtil.binaryScalaVersion(fv), + configurations = configurations, + checkExplicit = true, + filterImplicit = false, + overrideScalaVersion = overrideScalaVersion + ) + } + + val moduleSetting = ModuleDescriptorConfiguration(moduleId, ModuleInfo("foo")) + .withDependencies(deps) + .withConfigurations(configurations) + .withScalaModuleInfo(scalaModuleInfo) + lmEngine.moduleDescriptor(moduleSetting) + } + + def resolvers: Vector[Resolver] + +} diff --git a/coursier/src/test/scala/sbt/librarymanagement/coursier/ResolutionSpec.scala b/coursier/src/test/scala/sbt/librarymanagement/coursier/ResolutionSpec.scala new file mode 100644 index 000000000..29f8a9c95 --- /dev/null +++ b/coursier/src/test/scala/sbt/librarymanagement/coursier/ResolutionSpec.scala @@ -0,0 +1,142 @@ +package sbt.librarymanagement.coursier + +import sbt.librarymanagement.Configurations.Component +import sbt.librarymanagement.Resolver.{ + DefaultMavenRepository, + JCenterRepository, + JavaNet2Repository +} +import sbt.librarymanagement.syntax._ +import sbt.librarymanagement.{ Resolver, UnresolvedWarningConfiguration, UpdateConfiguration } + +class ResolutionSpec extends BaseCoursierSpecification { + override val resolvers = Vector( + DefaultMavenRepository, + JavaNet2Repository, + JCenterRepository, + Resolver.sbtPluginRepo("releases") + ) + + val lmEngine = new CoursierDependencyResolution(resolvers) + + private final val stubModule = "com.example" % "foo" % "0.1.0" % "compile" + + "Coursier dependency resolution" should "resolve very simple module" in { + val dependencies = Vector( + "com.typesafe.scala-logging" % "scala-logging_2.12" % "3.7.2" % "compile", + "org.scalatest" % "scalatest_2.12" % "3.0.4" % "test" + ).map(_.withIsTransitive(false)) + + val coursierModule = module(stubModule, dependencies, Some("2.12.4")) + val resolution = + lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) + + resolution should be('right) + val r = resolution.right.get + r.configurations.map(_.configuration) should have size 11 + + val compileConfig = r.configurations.find(_.configuration == Compile.toConfigRef).get + compileConfig.modules should have size 1 + + val runtimeConfig = r.configurations.find(_.configuration == Runtime.toConfigRef).get + runtimeConfig.modules should have size 1 + + val testConfig = r.configurations.find(_.configuration == Test.toConfigRef).get + testConfig.modules should have size 1 + } + + it should "resolve compiler bridge" in { + val dependencies = + Vector(("org.scala-sbt" % "compiler-interface" % "1.0.4" % "component").sources()) + val coursierModule = module(stubModule, dependencies, Some("2.12.4")) + val resolution = + lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) + + val r = resolution.right.get + + val componentConfig = r.configurations.find(_.configuration == Component.toConfigRef).get + componentConfig.modules should have size 1 + componentConfig.modules.head.artifacts should have size 1 + componentConfig.modules.head.artifacts.head._1.classifier should contain("sources") + } + + it should "resolve sbt jars" in { + val dependencies = + Vector(("org.scala-sbt" % "sbt" % "1.1.0" % "provided")) + val coursierModule = module(stubModule, dependencies, Some("2.12.4")) + val resolution = + lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) + + val r = resolution.right.get + + val modules = r.configurations.flatMap(_.modules) + modules.map(_.module.name) should contain("main_2.12") + } + + it should "resolve with default resolvers" in { + val dependencies = + Vector(("org.scala-sbt" % "compiler-interface" % "1.0.4" % "component").sources()) + val lmEngine = + CoursierDependencyResolution.apply(Resolver.combineDefaultResolvers(Vector.empty)) + val coursierModule = module(stubModule, dependencies, Some("2.12.4")) + val resolution = + lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) + + resolution should be('right) + } + + it should "resolve plugin" in { + val pluginAttributes = Map("scalaVersion" -> "2.12", "sbtVersion" -> "1.0") + val dependencies = + Vector(("org.xerial.sbt" % "sbt-sonatype" % "2.0").withExtraAttributes(pluginAttributes)) + val coursierModule = module(stubModule, dependencies, Some("2.12.4")) + val resolution = + lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) + + val r = resolution.right.get + + val componentConfig = r.configurations.find(_.configuration == Compile.toConfigRef).get + componentConfig.modules.map(_.module.name) should have size 5 + } + + it should "strip e: prefix from plugin attributes" in { + val pluginAttributes = Map("e:scalaVersion" -> "2.12", "e:sbtVersion" -> "1.0") + val dependencies = + Vector(("org.xerial.sbt" % "sbt-sonatype" % "2.0").withExtraAttributes(pluginAttributes)) + val coursierModule = module(stubModule, dependencies, Some("2.12.4")) + val resolution = + lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) + + resolution should be('right) + } + + it should "resolve plugins hosted on repo.typesafe.com" in { + val pluginAttributes = Map("e:scalaVersion" -> "2.12", "e:sbtVersion" -> "1.0") + val dependencies = + Vector(("com.typesafe.sbt" % "sbt-git" % "0.9.3").withExtraAttributes(pluginAttributes)) + val coursierModule = module(stubModule, dependencies, Some("2.12.4")) + val resolution = + lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log) + + resolution should be('right) + } + + it should "reorder fast and slow resolvers" in { + val resolvers = Vector( + JavaNet2Repository, + Resolver.sbtPluginRepo("releases"), + DefaultMavenRepository + ) + val engine = new CoursierDependencyResolution(resolvers) + engine.reorderedResolvers.head.name should be("public") + engine.reorderedResolvers.last.name should be("sbt-plugin-releases") + engine.reorderedResolvers should have size 3 + } + + it should "reorder default resolvers" in { + val resolvers = Resolver.combineDefaultResolvers(Vector.empty) + val engine = new CoursierDependencyResolution(resolvers) + engine.reorderedResolvers should not be 'empty + engine.reorderedResolvers.head.name should be("public") + } +} diff --git a/coursier/src/test/scala/sbt/librarymanagement/coursier/UnitSpec.scala b/coursier/src/test/scala/sbt/librarymanagement/coursier/UnitSpec.scala new file mode 100644 index 000000000..e6d7f8e75 --- /dev/null +++ b/coursier/src/test/scala/sbt/librarymanagement/coursier/UnitSpec.scala @@ -0,0 +1,5 @@ +package sbt.librarymanagement.coursier + +import org.scalatest.{ FlatSpec, Matchers } + +abstract class UnitSpec extends FlatSpec with Matchers diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 21ade5fc5..2e4d14634 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -6,8 +6,7 @@ object Dependencies { val scala211 = "2.11.12" val scala212 = "2.12.7" - private val ioVersion = "1.2.1" - private val utilVersion = "1.2.2" + private val coursierVersion = "1.1.0-M1" private val sbtIO = "org.scala-sbt" %% "io" % ioVersion @@ -41,7 +40,13 @@ object Dependencies { val launcherInterface = "org.scala-sbt" % "launcher-interface" % "1.0.0" val ivy = "org.scala-sbt.ivy" % "ivy" % "2.3.0-sbt-b18f59ea3bc914a297bb6f1a4f7fb0ace399e310" - val jsch = "com.jcraft" % "jsch" % "0.1.54" + val coursier = "io.get-coursier" %% "coursier" % coursierVersion + val coursierCache = "io.get-coursier" %% "coursier-cache" % coursierVersion + + val sbtV = "1.0" + val scalaV = "2.12" + + val jsch = "com.jcraft" % "jsch" % "0.1.54" intransitive () val scalaReflect = Def.setting { "org.scala-lang" % "scala-reflect" % scalaVersion.value } val scalaCompiler = Def.setting { "org.scala-lang" % "scala-compiler" % scalaVersion.value } val scalaXml = scala211Module("scala-xml", "1.0.5")