diff --git a/build.sbt b/build.sbt index e021f674d..87d993210 100644 --- a/build.sbt +++ b/build.sbt @@ -17,37 +17,33 @@ inThisBuild(List( val coursierVersion = "1.1.0-M8" -lazy val `sbt-shared` = project - .in(file("modules/sbt-shared")) +lazy val `lm-coursier` = project + .in(file("modules/lm-coursier")) .settings( shared, libraryDependencies ++= Seq( "io.get-coursier" %% "coursier" % coursierVersion, "io.get-coursier" %% "coursier-cache" % coursierVersion, - "org.scala-sbt" %% "librarymanagement-ivy" % "1.0.2" + "io.get-coursier" %% "coursier-extra" % coursierVersion, + "org.scala-sbt" %% "librarymanagement-core" % "1.0.2" ) ) lazy val `sbt-coursier` = project .in(file("modules/sbt-coursier")) .enablePlugins(ScriptedPlugin) - .dependsOn(`sbt-shared`) + .dependsOn(`lm-coursier`) .settings( plugin, libraryDependencies += "com.lihaoyi" %% "utest" % "0.6.4" % Test, testFrameworks += new TestFramework("utest.runner.Framework"), - libraryDependencies ++= Seq( - "io.get-coursier" %% "coursier" % coursierVersion, - "io.get-coursier" %% "coursier-cache" % coursierVersion, - "io.get-coursier" %% "coursier-extra" % coursierVersion, - "io.get-coursier" %% "coursier-scalaz-interop" % coursierVersion - ), + libraryDependencies +="io.get-coursier" %% "coursier-scalaz-interop" % coursierVersion, scriptedDependencies := { scriptedDependencies.value // TODO Get dependency projects automatically // (but shouldn't scripted itself handle that…?) - publishLocal.in(`sbt-shared`).value + publishLocal.in(`lm-coursier`).value } ) @@ -95,7 +91,7 @@ lazy val `sbt-shading` = project lazy val `sbt-coursier-root` = project .in(file(".")) .aggregate( - `sbt-shared`, + `lm-coursier`, `sbt-coursier`, `sbt-pgp-coursier`, `sbt-shading` diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsParams.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsParams.scala new file mode 100644 index 000000000..9d03f2e85 --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsParams.scala @@ -0,0 +1,22 @@ +package coursier.sbtcoursier + +import java.io.File + +import coursier.{Cache, CachePolicy} +import coursier.core.{Classifier, Resolution} + +import scala.concurrent.duration.Duration + +final case class ArtifactsParams( + classifiers: Option[Seq[Classifier]], + res: Seq[Resolution], + includeSignatures: Boolean, + parallelDownloads: Int, + createLogger: () => Cache.Logger, + cache: File, + artifactsChecksums: Seq[Option[String]], + ttl: Option[Duration], + cachePolicies: Seq[CachePolicy], + projectName: String, + sbtClassifiers: Boolean +) diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsRun.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsRun.scala new file mode 100644 index 000000000..d24dd5cfb --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsRun.scala @@ -0,0 +1,95 @@ +package coursier.sbtcoursier + +import java.io.File +import java.util.concurrent.ExecutorService + +import coursier.{Artifact, Cache, CachePolicy, FileError} +import coursier.util.{Schedulable, Task} +import sbt.util.Logger + +import scala.concurrent.ExecutionContext + +object ArtifactsRun { + + def artifacts( + params: ArtifactsParams, + verbosityLevel: Int, + log: Logger + ): Either[ResolutionError.UnknownDownloadException, Map[Artifact, Either[FileError, File]]] = { + + val allArtifacts0 = params.res.flatMap(_.dependencyArtifacts(params.classifiers)).map(_._3) + + val allArtifacts = + if (params.includeSignatures) + allArtifacts0.flatMap { a => + val sigOpt = a.extra.get("sig") + Seq(a) ++ sigOpt.toSeq + } + else + allArtifacts0 + + // let's update only one module at once, for a better output + // Downloads are already parallel, no need to parallelize further anyway + Lock.lock.synchronized { + + var pool: ExecutorService = null + var artifactsLogger: Cache.Logger = null + + val printOptionalMessage = verbosityLevel >= 0 && verbosityLevel <= 1 + + val artifactFilesOrErrors = try { + pool = Schedulable.fixedThreadPool(params.parallelDownloads) + artifactsLogger = params.createLogger() + + val artifactFileOrErrorTasks = allArtifacts.toVector.distinct.map { a => + def f(p: CachePolicy) = + Cache.file[Task]( + a, + params.cache, + p, + checksums = params.artifactsChecksums, + logger = Some(artifactsLogger), + pool = pool, + ttl = params.ttl + ) + + params.cachePolicies.tail + .foldLeft(f(params.cachePolicies.head))(_ orElse f(_)) + .run + .map((a, _)) + } + + val artifactInitialMessage = + if (verbosityLevel >= 0) + s"Fetching artifacts of ${params.projectName}" + + (if (params.sbtClassifiers) " (sbt classifiers)" else "") + else + "" + + if (verbosityLevel >= 2) + log.info(artifactInitialMessage) + + artifactsLogger.init(if (printOptionalMessage) log.info(artifactInitialMessage)) + + Task.gather.gather(artifactFileOrErrorTasks).attempt.unsafeRun()(ExecutionContext.fromExecutorService(pool)) match { + case Left(ex) => + Left(ResolutionError.UnknownDownloadException(ex)) + case Right(l) => + Right(l.toMap) + } + } finally { + if (pool != null) + pool.shutdown() + if (artifactsLogger != null) + if ((artifactsLogger.stopDidPrintSomething() && printOptionalMessage) || verbosityLevel >= 2) + log.info( + s"Fetched artifacts of ${params.projectName}" + + (if (params.sbtClassifiers) " (sbt classifiers)" else "") + ) + } + + artifactFilesOrErrors + } + } + +} diff --git a/modules/sbt-shared/src/main/scala/coursier/FromSbt.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/FromSbt.scala similarity index 97% rename from modules/sbt-shared/src/main/scala/coursier/FromSbt.scala rename to modules/lm-coursier/src/main/scala/coursier/sbtcoursier/FromSbt.scala index 2f85e9a76..f8f5e1ccc 100644 --- a/modules/sbt-shared/src/main/scala/coursier/FromSbt.scala +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/FromSbt.scala @@ -1,12 +1,14 @@ -package coursier +package coursier.sbtcoursier import coursier.ivy.IvyRepository import coursier.ivy.IvyXml.{mappings => ivyXmlMappings} import java.net.{MalformedURLException, URL} -import coursier.core.{Authentication, Classifier, Configuration, Type} +import coursier.{Attributes, Cache, Dependency, Module} +import coursier.core._ +import coursier.maven.MavenRepository import sbt.internal.librarymanagement.mavenint.SbtPomExtraProperties -import sbt.librarymanagement.{CrossVersion, FileRepository, GetClassifiersModule, ModuleID, Patterns, RawRepository, Resolver, URLRepository} +import sbt.librarymanagement.{Configuration => _, MavenRepository => _, _} import sbt.util.Logger object FromSbt { diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/Inputs.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/Inputs.scala new file mode 100644 index 000000000..e7175397d --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/Inputs.scala @@ -0,0 +1,142 @@ +package coursier.sbtcoursier + +import coursier.core.{Configuration, ModuleName, Organization, Project} +import sbt.librarymanagement.{InclExclRule, ModuleID} +import sbt.util.Logger + +import scala.collection.mutable + +object Inputs { + + def configExtends(configurations: Seq[sbt.librarymanagement.Configuration]): Map[Configuration, Seq[Configuration]] = + configurations + .map(cfg => Configuration(cfg.name) -> cfg.extendsConfigs.map(c => Configuration(c.name))) + .toMap + + def coursierConfigurations( + configurations: Seq[sbt.librarymanagement.Configuration], + shadedConfig: Option[(String, Configuration)] = None + ): Map[Configuration, Set[Configuration]] = { + + val configs0 = Inputs.configExtends(configurations) + + def allExtends(c: Configuration) = { + // possibly bad complexity + def helper(current: Set[Configuration]): Set[Configuration] = { + val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) + if ((newSet -- current).nonEmpty) + helper(newSet) + else + newSet + } + + helper(Set(c)) + } + + val map = configs0.map { + case (config, _) => + config -> allExtends(config) + } + + map ++ shadedConfig.toSeq.flatMap { + case (baseConfig, shadedConfig) => + val baseConfig0 = Configuration(baseConfig) + Seq( + baseConfig0 -> (map.getOrElse(baseConfig0, Set(baseConfig0)) + shadedConfig), + shadedConfig -> map.getOrElse(shadedConfig, Set(shadedConfig)) + ) + } + } + + def ivyGraphs(configurations: Map[Configuration, Seq[Configuration]]): Seq[Set[Configuration]] = { + + // probably bad complexity, but that shouldn't matter given the size of the graphs involved... + + final class Wrapper(val set: mutable.HashSet[Configuration]) { + def ++=(other: Wrapper): this.type = { + set ++= other.set + this + } + } + + val sets = + new mutable.HashMap[Configuration, Wrapper] ++= configurations.map { + case (k, l) => + val s = new mutable.HashSet[Configuration] + s ++= l + s += k + k -> new Wrapper(s) + } + + for (k <- configurations.keys) { + val s = sets(k) + + var foundNew = true + while (foundNew) { + foundNew = false + for (other <- s.set.toVector) { + val otherS = sets(other) + if (!otherS.eq(s)) { + s ++= otherS + sets += other -> s + foundNew = true + } + } + } + } + + sets.values.toVector.distinct.map(_.set.toSet) + } + + def coursierProject( + projId: ModuleID, + dependencies: Seq[ModuleID], + excludeDeps: Seq[InclExclRule], + configurations: Seq[sbt.librarymanagement.Configuration], + sv: String, + sbv: String, + log: Logger + ): Project = { + + val exclusions = { + + var anyNonSupportedExclusionRule = false + + val res = excludeDeps + .flatMap { rule => + if (rule.artifact != "*" || rule.configurations.nonEmpty) { + log.warn(s"Unsupported exclusion rule $rule") + anyNonSupportedExclusionRule = true + Nil + } else + Seq( + (Organization(rule.organization), ModuleName(FromSbt.sbtCrossVersionName(rule.name, rule.crossVersion, sv, sbv))) + ) + } + .toSet + + if (anyNonSupportedExclusionRule) + log.warn("Only supported exclusion rule fields: organization, name") + + res + } + + val configMap = configExtends(configurations) + + val proj = FromSbt.project( + projId, + dependencies, + configMap, + sv, + sbv + ) + + proj.copy( + dependencies = proj.dependencies.map { + case (config, dep) => + (config, dep.copy(exclusions = dep.exclusions ++ exclusions)) + } + ) + } + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/InterProjectRepository.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/InterProjectRepository.scala similarity index 89% rename from modules/sbt-coursier/src/main/scala/coursier/InterProjectRepository.scala rename to modules/lm-coursier/src/main/scala/coursier/sbtcoursier/InterProjectRepository.scala index ff6f0919d..e1226c4c9 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/InterProjectRepository.scala +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/InterProjectRepository.scala @@ -1,6 +1,7 @@ -package coursier +package coursier.sbtcoursier -import coursier.core.Classifier +import coursier.Fetch +import coursier.core._ import coursier.util.{EitherT, Monad} final case class InterProjectRepository(projects: Seq[Project]) extends Repository { diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/Lock.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/Lock.scala new file mode 100644 index 000000000..bbfb2c0a2 --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/Lock.scala @@ -0,0 +1,10 @@ +package coursier.sbtcoursier + +object Lock { + + // Wrap blocks downloading stuff (resolution / artifact downloads) in lock.synchronized. + // Downloads are already parallel, no need to parallelize further, and this results in + // a clearer output. + val lock = new Object + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/ResolutionError.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionError.scala similarity index 97% rename from modules/sbt-coursier/src/main/scala/coursier/ResolutionError.scala rename to modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionError.scala index 4dfc330b1..629f10a6f 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/ResolutionError.scala +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionError.scala @@ -1,4 +1,7 @@ -package coursier +package coursier.sbtcoursier + +import coursier.FileError +import coursier.core.Module import scala.collection.mutable.ArrayBuffer diff --git a/modules/sbt-coursier/src/main/scala/coursier/ResolutionException.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionException.scala similarity index 88% rename from modules/sbt-coursier/src/main/scala/coursier/ResolutionException.scala rename to modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionException.scala index 9f8925a91..a600328d1 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/ResolutionException.scala +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionException.scala @@ -1,4 +1,4 @@ -package coursier +package coursier.sbtcoursier final class ResolutionException( val error: ResolutionError diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionParams.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionParams.scala new file mode 100644 index 000000000..cf01b8a95 --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionParams.scala @@ -0,0 +1,216 @@ +package coursier.sbtcoursier + +import java.io.File +import java.net.URL + +import coursier.{Cache, CachePolicy, FallbackDependenciesRepository, ProjectCache, Resolution, moduleNameString} +import coursier.core._ +import coursier.extra.Typelevel +import coursier.ivy.PropertiesPattern +import sbt.librarymanagement.{Resolver, URLRepository} + +import scala.collection.mutable.ArrayBuffer +import scala.concurrent.duration.Duration + +final case class ResolutionParams( + dependencies: Seq[(Configuration, Dependency)], + fallbackDependencies: Seq[(Module, String, URL, Boolean)], + configGraphs: Seq[Set[Configuration]], + autoScalaLib: Boolean, + mainRepositories: Seq[Repository], + parentProjectCache: ProjectCache, + interProjectDependencies: Seq[Project], + internalRepositories: Seq[Repository], + userEnabledProfiles: Set[String], + userForceVersions: Map[Module, String], + typelevel: Boolean, + so: Organization, + sv: String, + sbtClassifiers: Boolean, + parallelDownloads: Int, + projectName: String, + maxIterations: Int, + createLogger: () => Cache.Logger, + cache: File, + cachePolicies: Seq[CachePolicy], + ttl: Option[Duration], + checksums: Seq[Option[String]] +) { + + val fallbackDependenciesRepositories = + if (fallbackDependencies.isEmpty) + Nil + else { + val map = fallbackDependencies.map { + case (mod, ver, url, changing) => + (mod, ver) -> ((url, changing)) + }.toMap + + Seq( + FallbackDependenciesRepository(map) + ) + } + + val repositories = + internalRepositories ++ + mainRepositories ++ + fallbackDependenciesRepositories + + private val noOptionalFilter: Option[Dependency => Boolean] = Some(dep => !dep.optional) + private val typelevelOrgSwap: Option[Dependency => Dependency] = Some(Typelevel.swap(_)) + + private def forcedScalaModules( + scalaOrganization: Organization, + scalaVersion: String + ): Map[Module, String] = + Map( + Module(scalaOrganization, name"scala-library", Map.empty) -> scalaVersion, + Module(scalaOrganization, name"scala-compiler", Map.empty) -> scalaVersion, + Module(scalaOrganization, name"scala-reflect", Map.empty) -> scalaVersion, + Module(scalaOrganization, name"scalap", Map.empty) -> scalaVersion + ) + + private def startRes(configs: Set[Configuration]) = Resolution( + dependencies + .collect { + case (config, dep) if configs(config) => + dep + } + .toSet, + filter = noOptionalFilter, + userActivations = + if (userEnabledProfiles.isEmpty) + None + else + Some(userEnabledProfiles.iterator.map(_ -> true).toMap), + forceVersions = + // order matters here + userForceVersions ++ + (if (autoScalaLib && (configs(Configuration.compile) || configs(Configuration("scala-tool")))) forcedScalaModules(so, sv) else Map()) ++ + interProjectDependencies.map(_.moduleVersion), + projectCache = parentProjectCache, + mapDependencies = if (typelevel && (configs(Configuration.compile) || configs(Configuration("scala-tool")))) typelevelOrgSwap else None + ) + + lazy val allStartRes = configGraphs.map(configs => configs -> startRes(configs)).toMap + + lazy val resolutionKey = SbtCoursierCache.ResolutionKey( + dependencies, + repositories, + userEnabledProfiles, + allStartRes, + sbtClassifiers + ) + +} + +object ResolutionParams { + + def defaultIvyProperties(): Map[String, String] = { + + val ivyHome = sys.props.getOrElse( + "ivy.home", + new File(sys.props("user.home")).toURI.getPath + ".ivy2" + ) + + val sbtIvyHome = sys.props.getOrElse( + "sbt.ivy.home", + ivyHome + ) + + Map( + "ivy.home" -> ivyHome, + "sbt.ivy.home" -> sbtIvyHome + ) ++ sys.props + } + + private def exceptionPatternParser(): String => coursier.ivy.Pattern = { + + val props = sys.props.toMap + + val extraProps = new ArrayBuffer[(String, String)] + + def addUriProp(key: String): Unit = + for (b <- props.get(key)) { + val uri = new File(b).toURI.toString + extraProps += s"$key.uri" -> uri + } + + addUriProp("sbt.global.base") + addUriProp("user.home") + + { + s => + val p = PropertiesPattern.parse(s) match { + case Left(err) => + throw new Exception(s"Cannot parse pattern $s: $err") + case Right(p) => + p + } + + p.substituteProperties(props ++ extraProps) match { + case Left(err) => + throw new Exception(err) + case Right(p) => + p + } + } + } + + def globalPluginPatterns(sbtVersion: String): Seq[coursier.ivy.Pattern] = { + + val defaultRawPattern = s"$${sbt.global.base.uri-$${user.home.uri}/.sbt/$sbtVersion}/plugins/target" + + "/resolution-cache/" + + "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]" + + // seems to be required in more recent versions of sbt (since 0.13.16?) + val extraRawPattern = s"$${sbt.global.base.uri-$${user.home.uri}/.sbt/$sbtVersion}/plugins/target" + + "(/scala-[scalaVersion])(/sbt-[sbtVersion])" + + "/resolution-cache/" + + "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]" + + val p = exceptionPatternParser() + + Seq( + defaultRawPattern, + extraRawPattern + ).map(p) + } + + private val slowReposBase = Seq( + "https://repo.typesafe.com/", + "https://repo.scala-sbt.org/", + "http://repo.typesafe.com/", + "http://repo.scala-sbt.org/" + ) + + private val fastReposBase = Seq( + "http://repo1.maven.org/", + "https://repo1.maven.org/" + ) + + private def url(res: Resolver): Option[String] = + res match { + case m: sbt.librarymanagement.MavenRepository => + Some(m.root) + case u: URLRepository => + u.patterns.artifactPatterns.headOption + .orElse(u.patterns.ivyPatterns.headOption) + case _ => + None + } + + private def fastRepo(res: Resolver): Boolean = + url(res).exists(u => fastReposBase.exists(u.startsWith)) + + private def slowRepo(res: Resolver): Boolean = + url(res).exists(u => slowReposBase.exists(u.startsWith)) + + def reorderResolvers(resolvers: Seq[Resolver]): Seq[Resolver] = + if (resolvers.exists(fastRepo) && resolvers.exists(slowRepo)) { + val (slow, other) = resolvers.partition(slowRepo) + other ++ slow + } else + resolvers + +} diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionRun.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionRun.scala new file mode 100644 index 000000000..1a7bef161 --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ResolutionRun.scala @@ -0,0 +1,169 @@ +package coursier.sbtcoursier + +import java.util.concurrent.ExecutorService + +import coursier.{Cache, Fetch, Resolution} +import coursier.core._ +import coursier.ivy.IvyRepository +import coursier.maven.MavenRepository +import coursier.util.{Print, Schedulable, Task} +import sbt.util.Logger + +import scala.concurrent.ExecutionContext + +object ResolutionRun { + + def resolution( + params: ResolutionParams, + verbosityLevel: Int, + log: Logger, + startRes: Resolution + ): Either[ResolutionError, Resolution] = { + + // TODO Re-use the thread pool across resolutions / downloads? + var pool: ExecutorService = null + + var resLogger: Cache.Logger = null + + val printOptionalMessage = verbosityLevel >= 0 && verbosityLevel <= 1 + + val resOrError: Either[ResolutionError, Resolution] = try { + pool = Schedulable.fixedThreadPool(params.parallelDownloads) + resLogger = params.createLogger() + + val fetch = Fetch.from( + params.repositories, + Cache.fetch[Task](params.cache, params.cachePolicies.head, checksums = params.checksums, logger = Some(resLogger), pool = pool, ttl = params.ttl), + params.cachePolicies.tail.map(p => + Cache.fetch[Task](params.cache, p, checksums = params.checksums, logger = Some(resLogger), pool = pool, ttl = params.ttl) + ): _* + ) + + def depsRepr(deps: Seq[(Configuration, Dependency)]) = + deps.map { case (config, dep) => + s"${dep.module}:${dep.version}:${config.value}->${dep.configuration.value}" + }.sorted.distinct + + if (verbosityLevel >= 2) { + val repoReprs = params.repositories.map { + case r: IvyRepository => + s"ivy:${r.pattern}" + case _: InterProjectRepository => + "inter-project" + case r: MavenRepository => + r.root + case r => + // should not happen + r.toString + } + + log.info( + "Repositories:\n" + + repoReprs.map(" " + _).mkString("\n") + ) + } + + val initialMessage = + Seq( + if (verbosityLevel >= 0) + Seq(s"Updating ${params.projectName}" + (if (params.sbtClassifiers) " (sbt classifiers)" else "")) + else + Nil, + if (verbosityLevel >= 2) + depsRepr(params.dependencies).map(depRepr => + s" $depRepr" + ) + else + Nil + ).flatten.mkString("\n") + + if (verbosityLevel >= 2) + log.info(initialMessage) + + resLogger.init(if (printOptionalMessage) log.info(initialMessage)) + + startRes + .process + .run(fetch, params.maxIterations) + .attempt + .unsafeRun()(ExecutionContext.fromExecutorService(pool)) + .left + .map(ex => + ResolutionError.UnknownException(ex) + ) + } finally { + if (pool != null) + pool.shutdown() + if (resLogger != null) + if ((resLogger.stopDidPrintSomething() && printOptionalMessage) || verbosityLevel >= 2) + log.info(s"Resolved ${params.projectName} dependencies") + } + + resOrError.flatMap { res => + if (!res.isDone) + Left( + ResolutionError.MaximumIterationsReached + ) + else if (res.conflicts.nonEmpty) { + val projCache = res.projectCache.mapValues { case (_, p) => p } + + Left( + ResolutionError.Conflicts( + "Conflict(s) in dependency resolution:\n " + + Print.dependenciesUnknownConfigs(res.conflicts.toVector, projCache) + ) + ) + } else if (res.errors.nonEmpty) { + val internalRepositoriesLen = params.internalRepositories.length + val errors = + if (params.repositories.length > internalRepositoriesLen) + // drop internal repository errors + res.errors.map { + case (dep, errs) => + dep -> errs.drop(internalRepositoriesLen) + } + else + res.errors + + Left( + ResolutionError.MetadataDownloadErrors(errors) + ) + } else + Right(res) + } + } + + def resolutions( + params: ResolutionParams, + verbosityLevel: Int, + log: Logger + ): Either[ResolutionError, Map[Set[Configuration], Resolution]] = { + + // TODO Warn about possible duplicated modules from source repositories? + + if (verbosityLevel >= 2) { + log.info("InterProjectRepository") + for (p <- params.interProjectDependencies) + log.info(s" ${p.module}:${p.version}") + } + + SbtCoursierCache.default.resolutionOpt(params.resolutionKey).map(Right(_)).getOrElse { + // Let's update only one module at once, for a better output. + // Downloads are already parallel, no need to parallelize further, anyway. + val resOrError = + Lock.lock.synchronized { + params.allStartRes.foldLeft[Either[ResolutionError, Map[Set[Configuration], Resolution]]](Right(Map())) { + case (acc, (config, startRes)) => + for { + m <- acc + res <- resolution(params, verbosityLevel, log, startRes) + } yield m + (config -> res) + } + } + for (res <- resOrError) + SbtCoursierCache.default.putResolution(params.resolutionKey, res) + resOrError + } + } + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/SbtBootJars.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/SbtBootJars.scala similarity index 72% rename from modules/sbt-coursier/src/main/scala/coursier/SbtBootJars.scala rename to modules/lm-coursier/src/main/scala/coursier/sbtcoursier/SbtBootJars.scala index 95fc92b5f..7adfae875 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/SbtBootJars.scala +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/SbtBootJars.scala @@ -1,7 +1,9 @@ -package coursier +package coursier.sbtcoursier import java.io.File +import coursier.core.{Module, ModuleName, Organization} + object SbtBootJars { def apply( scalaOrg: Organization, @@ -12,7 +14,7 @@ object SbtBootJars { .collect { case jar if jar.getName.endsWith(".jar") => val name = ModuleName(jar.getName.stripSuffix(".jar")) - val mod = Module(scalaOrg, name) + val mod = Module(scalaOrg, name, Map.empty) (mod, scalaVersion) -> jar } diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/SbtCoursierCache.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/SbtCoursierCache.scala new file mode 100644 index 000000000..e0de8ea5c --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/SbtCoursierCache.scala @@ -0,0 +1,59 @@ +package coursier.sbtcoursier + +import java.util.concurrent.ConcurrentHashMap + +import coursier.core._ +import sbt.librarymanagement.UpdateReport + +class SbtCoursierCache { + + import SbtCoursierCache._ + + private val resolutionsCache = new ConcurrentHashMap[ResolutionKey, Map[Set[Configuration], Resolution]] + // these may actually not need to be cached any more, now that the resolutions + // are cached + private val reportsCache = new ConcurrentHashMap[ReportKey, UpdateReport] + + + def resolutionOpt(key: ResolutionKey): Option[Map[Set[Configuration], Resolution]] = + Option(resolutionsCache.get(key)) + def putResolution(key: ResolutionKey, res: Map[Set[Configuration], Resolution]): Unit = + resolutionsCache.put(key, res) + + def reportOpt(key: ReportKey): Option[UpdateReport] = + Option(reportsCache.get(key)) + def putReport(key: ReportKey, report: UpdateReport): Unit = + reportsCache.put(key, report) + + def clear(): Unit = { + resolutionsCache.clear() + reportsCache.clear() + } + + def isEmpty: Boolean = + resolutionsCache.isEmpty && reportsCache.isEmpty + +} + +object SbtCoursierCache { + + final case class ResolutionKey( + dependencies: Seq[(Configuration, Dependency)], + repositories: Seq[Repository], + userEnabledProfiles: Set[String], + resolution: Map[Set[Configuration], Resolution], + sbtClassifiers: Boolean + ) + + final case class ReportKey( + dependencies: Seq[(Configuration, Dependency)], + resolution: Map[Set[Configuration], Resolution], + withClassifiers: Boolean, + sbtClassifiers: Boolean, + ignoreArtifactErrors: Boolean + ) + + + private[coursier] val default = new SbtCoursierCache + +} diff --git a/modules/sbt-shared/src/main/scala/coursier/ToSbt.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ToSbt.scala similarity index 97% rename from modules/sbt-shared/src/main/scala/coursier/ToSbt.scala rename to modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ToSbt.scala index 9edd6c511..99d73aaf8 100644 --- a/modules/sbt-shared/src/main/scala/coursier/ToSbt.scala +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/ToSbt.scala @@ -1,13 +1,14 @@ -package coursier +package coursier.sbtcoursier import java.io.File import java.net.URL import java.util.GregorianCalendar import java.util.concurrent.ConcurrentHashMap +import coursier.{Artifact, Attributes, Dependency, Module, Project, Resolution} import coursier.core.{Classifier, Configuration, Type} import coursier.maven.MavenAttributes -import sbt.librarymanagement.{Configuration => _, _} +import sbt.librarymanagement.{Artifact => _, Configuration => _, _} import sbt.util.Logger object ToSbt { @@ -253,7 +254,7 @@ object ToSbt { } UpdateReport( - null, + new File("."), configReports.toVector, UpdateStats(-1L, -1L, -1L, cached = false), Map.empty diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/UpdateParams.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/UpdateParams.scala new file mode 100644 index 000000000..fc7ee5b72 --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/UpdateParams.scala @@ -0,0 +1,18 @@ +package coursier.sbtcoursier + +import java.io.File + +import coursier.FileError +import coursier.core._ + +final case class UpdateParams( + shadedConfigOpt: Option[(String, Configuration)], + artifacts: Map[Artifact, Either[FileError, File]], + classifiers: Option[Seq[Classifier]], + configs: Map[Configuration, Set[Configuration]], + dependencies: Seq[(Configuration, Dependency)], + res: Map[Set[Configuration], Resolution], + ignoreArtifactErrors: Boolean, + includeSignatures: Boolean, + sbtBootJarOverrides: Map[(Module, String), File] +) diff --git a/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/UpdateRun.scala b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/UpdateRun.scala new file mode 100644 index 000000000..5bc233e20 --- /dev/null +++ b/modules/lm-coursier/src/main/scala/coursier/sbtcoursier/UpdateRun.scala @@ -0,0 +1,174 @@ +package coursier.sbtcoursier + +import java.io.File + +import coursier.core.Resolution.ModuleVersion +import coursier.core._ +import coursier.util.Print +import sbt.librarymanagement.UpdateReport +import sbt.util.Logger + +object UpdateRun { + + private def artifactFileOpt( + sbtBootJarOverrides: Map[(Module, String), File], + artifactFiles: Map[Artifact, File], + erroredArtifacts: Set[Artifact] + )( + module: Module, + version: String, + attributes: Attributes, + artifact: Artifact + ): Option[File] = { + + // Under some conditions, SBT puts the scala JARs of its own classpath + // in the application classpath. Ensuring we return SBT's jars rather than + // JARs from the coursier cache, so that a same JAR doesn't land twice in the + // application classpath (once via SBT jars, once via coursier cache). + val fromBootJars = + if (attributes.classifier.isEmpty && attributes.`type` == Type.jar) + sbtBootJarOverrides.get((module, version)) + else + None + + val res = fromBootJars.orElse(artifactFiles.get(artifact)) + + if (res.isEmpty && !erroredArtifacts(artifact)) + sys.error(s"${artifact.url} not downloaded (should not happen)") + + res + } + + // Move back to coursier.util (in core module) after 1.0? + private def allDependenciesByConfig( + res: Map[Configuration, Resolution], + depsByConfig: Map[Configuration, Set[Dependency]], + configs: Map[Configuration, Set[Configuration]] + ): Map[Configuration, Set[Dependency]] = { + + val allDepsByConfig = depsByConfig.map { + case (config, deps) => + config -> res(config).subset(deps).minDependencies + } + + val filteredAllDepsByConfig = allDepsByConfig.map { + case (config, allDeps) => + val allExtendedConfigs = configs.getOrElse(config, Set.empty) - config + val inherited = allExtendedConfigs + .flatMap(allDepsByConfig.getOrElse(_, Set.empty)) + + config -> (allDeps -- inherited) + } + + filteredAllDepsByConfig + } + + // Move back to coursier.util (in core module) after 1.0? + private def dependenciesWithConfig( + res: Map[Configuration, Resolution], + depsByConfig: Map[Configuration, Set[Dependency]], + configs: Map[Configuration, Set[Configuration]] + ): Set[Dependency] = + allDependenciesByConfig(res, depsByConfig, configs) + .flatMap { + case (config, deps) => + deps.map(dep => dep.copy(configuration = config --> dep.configuration)) + } + .groupBy(_.copy(configuration = Configuration.empty)) + .map { + case (dep, l) => + dep.copy(configuration = Configuration.join(l.map(_.configuration).toSeq: _*)) + } + .toSet + + def update( + params: UpdateParams, + verbosityLevel: Int, + log: Logger + ): Either[ResolutionError.DownloadErrors, UpdateReport] = { + + val configResolutions = params.res.flatMap { + case (configs, r) => + configs.iterator.map((_, r)) + } + + val depsByConfig = grouped(params.dependencies)( + config => + params.shadedConfigOpt match { + case Some((baseConfig, `config`)) => + Configuration(baseConfig) + case _ => + config + } + ) + + if (verbosityLevel >= 2) { + val finalDeps = dependenciesWithConfig( + configResolutions, + depsByConfig.map { case (k, l) => k -> l.toSet }, + params.configs + ) + + val projCache = params.res.values.foldLeft(Map.empty[ModuleVersion, Project])(_ ++ _.projectCache.mapValues(_._2)) + val repr = Print.dependenciesUnknownConfigs(finalDeps.toVector, projCache) + log.info(repr.split('\n').map(" " + _).mkString("\n")) + } + + val artifactFiles = params.artifacts.collect { + case (artifact, Right(file)) => + artifact -> file + } + + val artifactErrors = params + .artifacts + .toVector + .collect { + case (a, Left(err)) if !a.optional || !err.notFound => + a -> err + } + + // can be non empty only if ignoreArtifactErrors is true or some optional artifacts are not found + val erroredArtifacts = params + .artifacts + .collect { + case (artifact, Left(_)) => + artifact + } + .toSet + + def report = + ToSbt.updateReport( + depsByConfig, + configResolutions, + params.configs, + params.classifiers, + artifactFileOpt( + params.sbtBootJarOverrides, + artifactFiles, + erroredArtifacts + ), + log, + includeSignatures = params.includeSignatures + ) + + if (artifactErrors.isEmpty) + Right(report) + else { + val error = ResolutionError.DownloadErrors(artifactErrors.map(_._2)) + + if (params.ignoreArtifactErrors) { + log.warn(error.description(verbosityLevel >= 1)) + Right(report) + } else + Left(error) + } + } + + private def grouped[K, V](map: Seq[(K, V)])(mapKey: K => K): Map[K, Seq[V]] = + map + .groupBy(t => mapKey(t._1)) + .mapValues(_.map(_._2)) + .iterator + .toMap + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/Tasks.scala b/modules/sbt-coursier/src/main/scala/coursier/Tasks.scala deleted file mode 100644 index e860e8e68..000000000 --- a/modules/sbt-coursier/src/main/scala/coursier/Tasks.scala +++ /dev/null @@ -1,1463 +0,0 @@ -package coursier - -import java.io.File -import java.net.URL -import java.util.concurrent.{ConcurrentHashMap, ExecutorService, Executors} - -import coursier.core._ -import coursier.extra.Typelevel -import coursier.interop.scalaz._ -import coursier.ivy.{IvyRepository, PropertiesPattern} -import coursier.Keys._ -import coursier.Structure._ -import coursier.util.Print.Colors -import coursier.util.{Parse, Print} -import sbt.librarymanagement.{Configuration => _, _} -import sbt.{Classpaths, Def, Resolver, UpdateReport} -import sbt.Keys._ - -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer -import scala.util.Try -import scalaz.concurrent.{Strategy, Task} - -object Tasks { - - def allRecursiveInterDependencies(state: sbt.State, projectRef: sbt.ProjectRef) = { - - def dependencies(map: Map[String, Seq[String]], id: String): Set[String] = { - - def helper(map: Map[String, Seq[String]], acc: Set[String]): Set[String] = - if (acc.exists(map.contains)) { - val (kept, rem) = map.partition { case (k, _) => acc(k) } - helper(rem, acc ++ kept.valuesIterator.flatten) - } else - acc - - helper(map - id, map.getOrElse(id, Nil).toSet) - } - - val allProjectsDeps = - for (p <- structure(state).allProjects) - yield p.id -> p.dependencies.map(_.project.project) - - val deps = dependencies(allProjectsDeps.toMap, projectRef.project) - - structure(state).allProjectRefs.filter(p => deps(p.project)) - } - - private val slowReposBase = Seq( - "https://repo.typesafe.com/", - "https://repo.scala-sbt.org/", - "http://repo.typesafe.com/", - "http://repo.scala-sbt.org/" - ) - - private val fastReposBase = Seq( - "http://repo1.maven.org/", - "https://repo1.maven.org/" - ) - - def coursierResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = Def.taskDyn { - - def url(res: Resolver): Option[String] = - res match { - case m: sbt.librarymanagement.MavenRepository => - Some(m.root) - case u: sbt.URLRepository => - u.patterns.artifactPatterns.headOption - .orElse(u.patterns.ivyPatterns.headOption) - case _ => - None - } - - def fastRepo(res: Resolver): Boolean = - url(res).exists(u => fastReposBase.exists(u.startsWith)) - def slowRepo(res: Resolver): Boolean = - url(res).exists(u => slowReposBase.exists(u.startsWith)) - - val bootResOpt = bootResolvers.value - val overrideFlag = overrideBuildResolvers.value - - val resultTask = bootResOpt.filter(_ => overrideFlag) match { - case Some(r) => Def.task(r) - case None => - Def.taskDyn { - val extRes = externalResolvers.value - val isSbtPlugin = sbtPlugin.value - if (isSbtPlugin) - Def.task { - Seq( - sbtResolver.value, - Classpaths.sbtPluginReleases - ) ++ extRes - } - else - Def.task(extRes) - } - } - - Def.task { - val result = resultTask.value - val reorderResolvers = coursierReorderResolvers.value - val keepPreloaded = coursierKeepPreloaded.value - - val result0 = - if (reorderResolvers && result.exists(fastRepo) && result.exists(slowRepo)) { - val (slow, other) = result.partition(slowRepo) - other ++ slow - } else - result - - if (keepPreloaded) - result0 - else - result0.filter { r => - !r.name.startsWith("local-preloaded") - } - } - } - - def coursierRecursiveResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = - Def.taskDyn { - - val state = sbt.Keys.state.value - val projectRef = sbt.Keys.thisProjectRef.value - - val projects = allRecursiveInterDependencies(state, projectRef) - - val t = coursierResolvers - .forAllProjects(state, projectRef +: projects) - .map(_.values.toVector.flatten) - - Def.task(t.value) - } - - def coursierFallbackDependenciesTask: Def.Initialize[sbt.Task[Seq[(Module, String, URL, Boolean)]]] = - Def.taskDyn { - - val state = sbt.Keys.state.value - val projectRef = sbt.Keys.thisProjectRef.value - - val projects = allRecursiveInterDependencies(state, projectRef) - - val allDependenciesTask = allDependencies - .forAllProjects(state, projectRef +: projects) - .map(_.values.toVector.flatten) - - Def.task { - val allDependencies = allDependenciesTask.value - - FromSbt.fallbackDependencies( - allDependencies, - scalaVersion.in(projectRef).get(state), - scalaBinaryVersion.in(projectRef).get(state) - ) - } - } - - def coursierProjectTask: Def.Initialize[sbt.Task[Project]] = - Def.taskDyn { - - val state = sbt.Keys.state.value - val projectRef = sbt.Keys.thisProjectRef.value - - // should projectID.configurations be used instead? - val configurations = ivyConfigurations.in(projectRef).get(state) - - val allDependenciesTask = allDependencies.in(projectRef).get(state) - - lazy val projId = projectID.in(projectRef).get(state) - lazy val sv = scalaVersion.in(projectRef).get(state) - lazy val sbv = scalaBinaryVersion.in(projectRef).get(state) - - lazy val exclusions = { - - var anyNonSupportedExclusionRule = false - - val res = excludeDependencies - .in(projectRef) - .get(state) - .flatMap { rule => - if (rule.artifact != "*" || rule.configurations.nonEmpty) { - state.log.warn(s"Unsupported exclusion rule $rule") - anyNonSupportedExclusionRule = true - Nil - } else - Seq( - (Organization(rule.organization), ModuleName(FromSbt.sbtCrossVersionName(rule.name, rule.crossVersion, sv, sbv))) - ) - } - .toSet - - if (anyNonSupportedExclusionRule) - state.log.warn("Only supported exclusion rule fields: organization, name") - - res - } - - Def.task { - - val allDependencies = allDependenciesTask.value - - val configMap = configurations - .map(cfg => Configuration(cfg.name) -> cfg.extendsConfigs.map(c => Configuration(c.name))) - .toMap - - val proj = FromSbt.project( - projId, - allDependencies, - configMap, - sv, - sbv - ) - - proj.copy( - dependencies = proj.dependencies.map { - case (config, dep) => - (config, dep.copy(exclusions = dep.exclusions ++ exclusions)) - } - ) - } - } - - def coursierInterProjectDependenciesTask: Def.Initialize[sbt.Task[Seq[Project]]] = - Def.taskDyn { - - val state = sbt.Keys.state.value - val projectRef = sbt.Keys.thisProjectRef.value - - val projects = allRecursiveInterDependencies(state, projectRef) - - val t = coursierProject.forAllProjects(state, projects).map(_.values.toVector) - - Def.task(t.value) - } - - def coursierPublicationsTask( - configsMap: (sbt.Configuration, Configuration)* - ): Def.Initialize[sbt.Task[Seq[(Configuration, Publication)]]] = - Def.task { - - val state = sbt.Keys.state.value - val projectRef = sbt.Keys.thisProjectRef.value - val projId = sbt.Keys.projectID.value - val sv = sbt.Keys.scalaVersion.value - val sbv = sbt.Keys.scalaBinaryVersion.value - val ivyConfs = sbt.Keys.ivyConfigurations.value - - val sourcesConfigOpt = - if (ivyConfigurations.value.exists(_.name == "sources")) - Some(Configuration("sources")) - else - None - - val docsConfigOpt = - if (ivyConfigurations.value.exists(_.name == "docs")) - Some(Configuration("docs")) - else - None - - val sbtBinArtifacts = - for ((config, targetConfig) <- configsMap) yield { - - val publish = publishArtifact - .in(projectRef) - .in(packageBin) - .in(config) - .getOrElse(state, false) - - if (publish) - artifact - .in(projectRef) - .in(packageBin) - .in(config) - .find(state) - .map(targetConfig -> _) - else - None - } - - val sbtSourceArtifacts = - for ((config, targetConfig) <- configsMap) yield { - - val publish = publishArtifact - .in(projectRef) - .in(packageSrc) - .in(config) - .getOrElse(state, false) - - if (publish) - artifact - .in(projectRef) - .in(packageSrc) - .in(config) - .find(state) - .map(sourcesConfigOpt.getOrElse(targetConfig) -> _) - else - None - } - - val sbtDocArtifacts = - for ((config, targetConfig) <- configsMap) yield { - - val publish = publishArtifact - .in(projectRef) - .in(packageDoc) - .in(config) - .getOrElse(state, false) - - if (publish) - artifact - .in(projectRef) - .in(packageDoc) - .in(config) - .find(state) - .map(docsConfigOpt.getOrElse(targetConfig) -> _) - else - None - } - - val sbtArtifacts = sbtBinArtifacts ++ sbtSourceArtifacts ++ sbtDocArtifacts - - def artifactPublication(artifact: sbt.Artifact) = { - - val name = FromSbt.sbtCrossVersionName( - artifact.name, - projId.crossVersion, - sv, - sbv - ) - - Publication( - name, - Type(artifact.`type`), - Extension(artifact.extension), - artifact.classifier.fold(Classifier.empty)(Classifier(_)) - ) - } - - val sbtArtifactsPublication = sbtArtifacts.collect { - case Some((config, artifact)) => - config -> artifactPublication(artifact) - } - - val stdArtifactsSet = sbtArtifacts.flatMap(_.map { case (_, a) => a }.toSeq).toSet - - // Second-way of getting artifacts from SBT - // No obvious way of getting the corresponding publishArtifact value for the ones - // only here, it seems. - val extraSbtArtifacts = artifacts.in(projectRef).getOrElse(state, Nil) - .filterNot(stdArtifactsSet) - - // Seems that SBT does that - if an artifact has no configs, - // it puts it in all of them. See for example what happens to - // the standalone JAR artifact of the coursier cli module. - def allConfigsIfEmpty(configs: Iterable[ConfigRef]): Iterable[ConfigRef] = - if (configs.isEmpty) ivyConfs.filter(_.isPublic).map(c => ConfigRef(c.name)) else configs - - val extraSbtArtifactsPublication = for { - artifact <- extraSbtArtifacts - config <- allConfigsIfEmpty(artifact.configurations.map(x => ConfigRef(x.name))) - // FIXME If some configurations from artifact.configurations are not public, they may leak here :\ - } yield Configuration(config.name) -> artifactPublication(artifact) - - sbtArtifactsPublication ++ extraSbtArtifactsPublication - } - - def coursierConfigurationsTask(shadedConfig: Option[(String, Configuration)]) = Def.task { - - val configs0 = ivyConfigurations - .value - .map { config => - Configuration(config.name) -> config.extendsConfigs.map(c => Configuration(c.name)) - } - .toMap - - def allExtends(c: Configuration) = { - // possibly bad complexity - def helper(current: Set[Configuration]): Set[Configuration] = { - val newSet = current ++ current.flatMap(configs0.getOrElse(_, Nil)) - if ((newSet -- current).nonEmpty) - helper(newSet) - else - newSet - } - - helper(Set(c)) - } - - val map = configs0.map { - case (config, _) => - config -> allExtends(config) - } - - map ++ shadedConfig.toSeq.flatMap { - case (baseConfig, shadedConfig) => - val baseConfig0 = Configuration(baseConfig) - Seq( - baseConfig0 -> (map.getOrElse(baseConfig0, Set(baseConfig0)) + shadedConfig), - shadedConfig -> map.getOrElse(shadedConfig, Set(shadedConfig)) - ) - } - } - - private[coursier] final case class ResolutionCacheKey( - project: Project, - repositories: Seq[Repository], - userEnabledProfiles: Set[String], - resolution: Map[Set[Configuration], Resolution], - sbtClassifiers: Boolean - ) - - private[coursier] final case class ReportCacheKey( - project: Project, - resolution: Map[Set[Configuration], Resolution], - withClassifiers: Boolean, - sbtClassifiers: Boolean, - ignoreArtifactErrors: Boolean - ) - - private[coursier] val resolutionsCache = new ConcurrentHashMap[ResolutionCacheKey, Map[Set[Configuration], Resolution]] - // these may actually not need to be cached any more, now that the resolutions - // are cached - private[coursier] val reportsCache = new ConcurrentHashMap[ReportCacheKey, UpdateReport] - - private def forcedScalaModules( - scalaOrganization: Organization, - scalaVersion: String - ): Map[Module, String] = - Map( - Module(scalaOrganization, name"scala-library") -> scalaVersion, - Module(scalaOrganization, name"scala-compiler") -> scalaVersion, - Module(scalaOrganization, name"scala-reflect") -> scalaVersion, - Module(scalaOrganization, name"scalap") -> scalaVersion - ) - - private[coursier] def exceptionPatternParser(): String => coursier.ivy.Pattern = { - - val props = sys.props.toMap - - val extraProps = new ArrayBuffer[(String, String)] - - def addUriProp(key: String): Unit = - for (b <- props.get(key)) { - val uri = new File(b).toURI.toString - extraProps += s"$key.uri" -> uri - } - - addUriProp("sbt.global.base") - addUriProp("user.home") - - { - s => - val p = PropertiesPattern.parse(s) match { - case Left(err) => - throw new Exception(s"Cannot parse pattern $s: $err") - case Right(p) => - p - } - - p.substituteProperties(props ++ extraProps) match { - case Left(err) => - throw new Exception(err) - case Right(p) => - p - } - } - } - - private def globalPluginPatterns(sbtVersion: String): Seq[coursier.ivy.Pattern] = { - - // FIXME get the 0.13 automatically? - val defaultRawPattern = s"$${sbt.global.base.uri-$${user.home.uri}/.sbt/$sbtVersion}/plugins/target" + - "/resolution-cache/" + - "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]" - - // seems to be required in more recent versions of sbt (since 0.13.16?) - val extraRawPattern = s"$${sbt.global.base.uri-$${user.home.uri}/.sbt/$sbtVersion}/plugins/target" + - "(/scala-[scalaVersion])(/sbt-[sbtVersion])" + - "/resolution-cache/" + - "[organization]/[module](/scala_[scalaVersion])(/sbt_[sbtVersion])/[revision]/resolved.xml.[ext]" - - Seq( - defaultRawPattern, - extraRawPattern - ).map(exceptionPatternParser()) - } - - def parentProjectCacheTask: Def.Initialize[sbt.Task[Map[Seq[sbt.Resolver],Seq[coursier.ProjectCache]]]] = - Def.taskDyn { - - val state = sbt.Keys.state.value - val projectRef = sbt.Keys.thisProjectRef.value - - val projectDeps = structure(state).allProjects - .find(_.id == projectRef.project) - .map(_.dependencies.map(_.project.project).toSet) - .getOrElse(Set.empty) - - val projects = structure(state).allProjectRefs.filter(p => projectDeps(p.project)) - - val t = - for { - m <- coursierRecursiveResolvers.forAllProjects(state, projects) - n <- coursierResolutions.forAllProjects(state, m.keys.toSeq) - } yield - n.foldLeft(Map.empty[Seq[Resolver], Seq[ProjectCache]]) { - case (caches, (ref, resolutions)) => - val mainResOpt = resolutions.collectFirst { - case (k, v) if k(Configuration.compile) => v - } - - val r = for { - resolvers <- m.get(ref) - resolution <- mainResOpt - } yield - caches.updated(resolvers, resolution.projectCache +: caches.getOrElse(resolvers, Seq.empty)) - - r.getOrElse(caches) - } - - Def.task(t.value) - } - - - def ivyGraphsTask = Def.task { - - // probably bad complexity, but that shouldn't matter given the size of the graphs involved... - - val p = coursierProject.value - - final class Wrapper(val set: mutable.HashSet[Configuration]) { - def ++=(other: Wrapper): this.type = { - set ++= other.set - this - } - } - - val sets = - new mutable.HashMap[Configuration, Wrapper] ++= p.configurations.map { - case (k, l) => - val s = new mutable.HashSet[Configuration] - s ++= l - s += k - k -> new Wrapper(s) - } - - for (k <- p.configurations.keys) { - val s = sets(k) - - var foundNew = true - while (foundNew) { - foundNew = false - for (other <- s.set.toVector) { - val otherS = sets(other) - if (!otherS.eq(s)) { - s ++= otherS - sets += other -> s - foundNew = true - } - } - } - } - - sets.values.toVector.distinct.map(_.set.toSet) - } - - private val noOptionalFilter: Option[Dependency => Boolean] = Some(dep => !dep.optional) - private val typelevelOrgSwap: Option[Dependency => Dependency] = Some(Typelevel.swap(_)) - - - def resolutionsTask( - sbtClassifiers: Boolean = false - ): Def.Initialize[sbt.Task[Map[Set[Configuration], coursier.Resolution]]] = Def.taskDyn { - val projectName = thisProjectRef.value.project - - val sv = scalaVersion.value - val sbv = scalaBinaryVersion.value - - val currentProjectTask: sbt.Def.Initialize[sbt.Task[(Project, Seq[(Module, String, URL, Boolean)], Seq[Set[Configuration]])]] = - if (sbtClassifiers) - Def.task { - val cm = coursierSbtClassifiersModule.value - val proj = FromSbt.sbtClassifiersProject(cm, sv, sbv) - - val fallbackDeps = FromSbt.fallbackDependencies( - cm.dependencies, - sv, - sbv - ) - - (proj, fallbackDeps, Vector(cm.configurations.map(c => Configuration(c.name)).toSet)) - } - else - Def.task { - val baseConfigGraphs = coursierConfigGraphs.value - (coursierProject.value.copy(publications = coursierPublications.value), coursierFallbackDependencies.value, baseConfigGraphs) - } - - val interProjectDependencies = coursierInterProjectDependencies.value - - val parallelDownloads = coursierParallelDownloads.value - val checksums = coursierChecksums.value - val maxIterations = coursierMaxIterations.value - val cachePolicies = coursierCachePolicies.value - val ttl = coursierTtl.value - val cache = coursierCache.value - val createLogger = coursierCreateLogger.value - - val log = streams.value.log - - // are these always defined? (e.g. for Java only projects?) - val so = Organization(scalaOrganization.value) - - val userForceVersions = dependencyOverrides - .value - .map(FromSbt.moduleVersion(_, sv, sbv)) - .toMap - - val resolversTask = - if (sbtClassifiers) - Def.task(coursierSbtResolvers.value) - else - Def.task(coursierRecursiveResolvers.value.distinct) - - val verbosityLevel = coursierVerbosity.value - - val userEnabledProfiles = mavenProfiles.value - - val typelevel = Organization(scalaOrganization.value) == Typelevel.typelevelOrg - - val globalPluginsRepos = - for (p <- globalPluginPatterns(sbtBinaryVersion.value)) - yield IvyRepository.fromPattern( - p, - withChecksums = false, - withSignatures = false, - withArtifacts = false - ) - - val interProjectRepo = InterProjectRepository(interProjectDependencies) - - val ivyHome = sys.props.getOrElse( - "ivy.home", - new File(sys.props("user.home")).toURI.getPath + ".ivy2" - ) - - val sbtIvyHome = sys.props.getOrElse( - "sbt.ivy.home", - ivyHome - ) - - val ivyProperties = Map( - "ivy.home" -> ivyHome, - "sbt.ivy.home" -> sbtIvyHome - ) ++ sys.props - - val useSbtCredentials = coursierUseSbtCredentials.value - - val authenticationByHostTask = - if (useSbtCredentials) - Def.task { - sbt.Keys.credentials.value - .flatMap { - case dc: sbt.DirectCredentials => List(dc) - case fc: sbt.FileCredentials => - sbt.Credentials.loadCredentials(fc.path) match { - case Left(err) => - log.warn(s"$err, ignoring it") - Nil - case Right(dc) => List(dc) - } - } - .map { c => - c.host -> Authentication(c.userName, c.passwd) - } - .toMap - } - else - Def.task(Map.empty[String, Authentication]) - - val authenticationByRepositoryId = coursierCredentials.value.mapValues(_.authentication) - - def resTask( - currentProject: Project, - fallbackDependencies: Seq[(Module, String, URL, Boolean)], - configGraphs: Seq[Set[Configuration]], - repositories: Seq[Repository], - internalRepositories: Seq[Repository], - allStartRes: Map[Set[Configuration], coursier.Resolution] - ) = Def.task { - - def resolution(startRes: Resolution) = { - - var pool: ExecutorService = null - var resLogger: Cache.Logger = null - - val printOptionalMessage = verbosityLevel >= 0 && verbosityLevel <= 1 - - val res = try { - pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) - resLogger = createLogger() - - val fetch = Fetch.from( - repositories, - Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl), - cachePolicies.tail.map(p => - Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl) - ): _* - ) - - def depsRepr(deps: Seq[(Configuration, Dependency)]) = - deps.map { case (config, dep) => - s"${dep.module}:${dep.version}:${config.value}->${dep.configuration.value}" - }.sorted.distinct - - if (verbosityLevel >= 2) { - val repoReprs = repositories.map { - case r: IvyRepository => - s"ivy:${r.pattern}" - case r: InterProjectRepository => - "inter-project" - case r: MavenRepository => - r.root - case r => - // should not happen - r.toString - } - - log.info( - "Repositories:\n" + - repoReprs.map(" " + _).mkString("\n") - ) - } - - val initialMessage = - Seq( - if (verbosityLevel >= 0) - Seq(s"Updating $projectName" + (if (sbtClassifiers) " (sbt classifiers)" else "")) - else - Nil, - if (verbosityLevel >= 2) - depsRepr(currentProject.dependencies).map(depRepr => - s" $depRepr" - ) - else - Nil - ).flatten.mkString("\n") - - if (verbosityLevel >= 2) - log.info(initialMessage) - - resLogger.init(if (printOptionalMessage) log.info(initialMessage)) - - startRes - .process - .run(fetch, maxIterations) - .unsafePerformSyncAttempt - .toEither - .left - .map(ex => - ResolutionError.UnknownException(ex) - .throwException() - ) - .merge - } finally { - if (pool != null) - pool.shutdown() - if (resLogger != null) - if ((resLogger.stopDidPrintSomething() && printOptionalMessage) || verbosityLevel >= 2) - log.info(s"Resolved $projectName dependencies") - } - - if (!res.isDone) - ResolutionError.MaximumIterationsReached - .throwException() - - if (res.conflicts.nonEmpty) { - val projCache = res.projectCache.mapValues { case (_, p) => p } - - ResolutionError.Conflicts( - "Conflict(s) in dependency resolution:\n " + - Print.dependenciesUnknownConfigs(res.conflicts.toVector, projCache) - ).throwException() - } - - if (res.errors.nonEmpty) { - val internalRepositoriesLen = internalRepositories.length - val errors = - if (repositories.length > internalRepositoriesLen) - // drop internal repository errors - res.errors.map { - case (dep, errs) => - dep -> errs.drop(internalRepositoriesLen) - } - else - res.errors - - ResolutionError.MetadataDownloadErrors(errors) - .throwException() - } - - res - } - - // let's update only one module at once, for a better output - // Downloads are already parallel, no need to parallelize further anyway - synchronized { - allStartRes.map { - case (config, startRes) => - config -> resolution(startRes) - } - } - } - - Def.taskDyn { - - val (currentProject, fallbackDependencies, configGraphs) = currentProjectTask.value - - val autoScalaLib = autoScalaLibrary.value - - val resolvers = resolversTask.value - - // TODO Warn about possible duplicated modules from source repositories? - - val authenticationByHost = authenticationByHostTask.value - - val fallbackDependenciesRepositories = - if (fallbackDependencies.isEmpty) - Nil - else { - val map = fallbackDependencies.map { - case (mod, ver, url, changing) => - (mod, ver) -> ((url, changing)) - }.toMap - - Seq( - FallbackDependenciesRepository(map) - ) - } - - if (verbosityLevel >= 2) { - log.info("InterProjectRepository") - for (p <- interProjectDependencies) - log.info(s" ${p.module}:${p.version}") - } - - def withAuthenticationByHost(repo: Repository, credentials: Map[String, Authentication]): Repository = { - - def httpHost(s: String) = - if (s.startsWith("http://") || s.startsWith("https://")) - Try(Cache.url(s).getHost).toOption - else - None - - repo match { - case m: MavenRepository => - if (m.authentication.isEmpty) - httpHost(m.root).flatMap(credentials.get).fold(m) { auth => - m.copy(authentication = Some(auth)) - } - else - m - case i: IvyRepository => - if (i.authentication.isEmpty) { - val base = i.pattern.chunks.takeWhile { - case _: coursier.ivy.Pattern.Chunk.Const => true - case _ => false - }.map(_.string).mkString - - httpHost(base).flatMap(credentials.get).fold(i) { auth => - i.copy(authentication = Some(auth)) - } - } else - i - case _ => - repo - } - } - - val internalRepositories = globalPluginsRepos :+ interProjectRepo - - val repositories = - internalRepositories ++ - resolvers.flatMap { resolver => - FromSbt.repository( - resolver, - ivyProperties, - log, - authenticationByRepositoryId.get(resolver.name) - ) - }.map(withAuthenticationByHost(_, authenticationByHost)) ++ - fallbackDependenciesRepositories - - val parentProjectCache: ProjectCache = coursierParentProjectCache.value - .get(resolvers) - .map(_.foldLeft[ProjectCache](Map.empty)(_ ++ _)) - .getOrElse(Map.empty) - - def startRes(configs: Set[Configuration]) = Resolution( - currentProject - .dependencies - .collect { - case (config, dep) if configs(config) => - dep - } - .toSet, - filter = noOptionalFilter, - userActivations = - if (userEnabledProfiles.isEmpty) - None - else - Some(userEnabledProfiles.iterator.map(_ -> true).toMap), - forceVersions = - // order matters here - userForceVersions ++ - (if (autoScalaLib && (configs(Configuration.compile) || configs(Configuration("scala-tool")))) forcedScalaModules(so, sv) else Map()) ++ - interProjectDependencies.map(_.moduleVersion), - projectCache = parentProjectCache, - mapDependencies = if (typelevel && (configs(Configuration.compile) || configs(Configuration("scala-tool")))) typelevelOrgSwap else None - ) - - val allStartRes = configGraphs.map(configs => configs -> startRes(configs)).toMap - - val key = ResolutionCacheKey( - currentProject, - repositories, - userEnabledProfiles, - allStartRes, - sbtClassifiers - ) - - Option(resolutionsCache.get(key)) match { - case Some(res) => - Def.task(res) - case None => - val t = resTask( - currentProject, - fallbackDependencies, - configGraphs, - repositories, - internalRepositories, - allStartRes - ) - - t.map { res => - resolutionsCache.put(key, res) - res - } - } - } - } - - def artifactFilesOrErrors( - withClassifiers: Boolean, - sbtClassifiers: Boolean = false, - ignoreArtifactErrors: Boolean = false, - includeSignatures: Boolean = false - ) = Def.taskDyn { - val projectName = thisProjectRef.value.project - - val parallelDownloads = coursierParallelDownloads.value - val artifactsChecksums = coursierArtifactsChecksums.value - val cachePolicies = coursierCachePolicies.value - val ttl = coursierTtl.value - val cache = coursierCache.value - val createLogger = coursierCreateLogger.value - - val log = streams.value.log - - val verbosityLevel = coursierVerbosity.value - - val resTask: sbt.Def.Initialize[sbt.Task[Seq[Resolution]]] = - if (withClassifiers && sbtClassifiers) - Def.task(Seq(coursierSbtClassifiersResolution.value)) - else - Def.task(coursierResolutions.value.values.toVector) - - val classifiersTask: sbt.Def.Initialize[sbt.Task[Option[Seq[Classifier]]]] = - if (withClassifiers) { - if (sbtClassifiers) - Def.task(Some(coursierSbtClassifiersModule.value.classifiers.map(Classifier(_)))) - else - Def.task(Some(transitiveClassifiers.value.map(Classifier(_)))) - } else - Def.task(None) - - Def.task { - - val classifiers = classifiersTask.value - val res = resTask.value - - val allArtifacts0 = res.flatMap(_.dependencyArtifacts(classifiers)).map(_._3) - - val allArtifacts = - if (includeSignatures) - allArtifacts0.flatMap { a => - val sigOpt = a.extra.get("sig") - Seq(a) ++ sigOpt.toSeq - } - else - allArtifacts0 - - // let's update only one module at once, for a better output - // Downloads are already parallel, no need to parallelize further anyway - synchronized { - - var pool: ExecutorService = null - var artifactsLogger: Cache.Logger = null - - val printOptionalMessage = verbosityLevel >= 0 && verbosityLevel <= 1 - - val artifactFilesOrErrors = try { - pool = Executors.newFixedThreadPool(parallelDownloads, Strategy.DefaultDaemonThreadFactory) - artifactsLogger = createLogger() - - val artifactFileOrErrorTasks = allArtifacts.toVector.distinct.map { a => - def f(p: CachePolicy) = - Cache.file[Task]( - a, - cache, - p, - checksums = artifactsChecksums, - logger = Some(artifactsLogger), - pool = pool, - ttl = ttl - ) - - cachePolicies.tail - .foldLeft(f(cachePolicies.head))(_ orElse f(_)) - .run - .map((a, _)) - } - - val artifactInitialMessage = - if (verbosityLevel >= 0) - s"Fetching artifacts of $projectName" + - (if (sbtClassifiers) " (sbt classifiers)" else "") - else - "" - - if (verbosityLevel >= 2) - log.info(artifactInitialMessage) - - artifactsLogger.init(if (printOptionalMessage) log.info(artifactInitialMessage)) - - Task.gatherUnordered(artifactFileOrErrorTasks).unsafePerformSyncAttempt.toEither match { - case Left(ex) => - ResolutionError.UnknownDownloadException(ex) - .throwException() - case Right(l) => - l.toMap - } - } finally { - if (pool != null) - pool.shutdown() - if (artifactsLogger != null) - if ((artifactsLogger.stopDidPrintSomething() && printOptionalMessage) || verbosityLevel >= 2) - log.info( - s"Fetched artifacts of $projectName" + - (if (sbtClassifiers) " (sbt classifiers)" else "") - ) - } - - artifactFilesOrErrors - } - } - } - - private def artifactFileOpt( - sbtBootJarOverrides: Map[(Module, String), File], - artifactFiles: Map[Artifact, File], - erroredArtifacts: Set[Artifact], - log: sbt.Logger, - module: Module, - version: String, - attributes: Attributes, - artifact: Artifact - ) = { - - // Under some conditions, SBT puts the scala JARs of its own classpath - // in the application classpath. Ensuring we return SBT's jars rather than - // JARs from the coursier cache, so that a same JAR doesn't land twice in the - // application classpath (once via SBT jars, once via coursier cache). - val fromBootJars = - if (attributes.classifier.isEmpty && attributes.`type` == Type.jar) - sbtBootJarOverrides.get((module, version)) - else - None - - val res = fromBootJars.orElse(artifactFiles.get(artifact)) - - if (res.isEmpty && !erroredArtifacts(artifact)) - sys.error(s"${artifact.url} not downloaded (should not happen)") - - res - } - - // Move back to coursier.util (in core module) after 1.0? - private def allDependenciesByConfig( - res: Map[Configuration, Resolution], - depsByConfig: Map[Configuration, Set[Dependency]], - configs: Map[Configuration, Set[Configuration]] - ): Map[Configuration, Set[Dependency]] = { - - val allDepsByConfig = depsByConfig.map { - case (config, deps) => - config -> res(config).subset(deps).minDependencies - } - - val filteredAllDepsByConfig = allDepsByConfig.map { - case (config, allDeps) => - val allExtendedConfigs = configs.getOrElse(config, Set.empty) - config - val inherited = allExtendedConfigs - .flatMap(allDepsByConfig.getOrElse(_, Set.empty)) - - config -> (allDeps -- inherited) - } - - filteredAllDepsByConfig - } - - // Move back to coursier.util (in core module) after 1.0? - private def dependenciesWithConfig( - res: Map[Configuration, Resolution], - depsByConfig: Map[Configuration, Set[Dependency]], - configs: Map[Configuration, Set[Configuration]] - ): Set[Dependency] = - allDependenciesByConfig(res, depsByConfig, configs) - .flatMap { - case (config, deps) => - deps.map(dep => dep.copy(configuration = config --> dep.configuration)) - } - .groupBy(_.copy(configuration = Configuration.empty)) - .map { - case (dep, l) => - dep.copy(configuration = Configuration.join(l.map(_.configuration).toSeq: _*)) - } - .toSet - - def updateTask( - shadedConfigOpt: Option[(String, Configuration)], - withClassifiers: Boolean, - sbtClassifiers: Boolean = false, - ignoreArtifactErrors: Boolean = false, - includeSignatures: Boolean = false - ): Def.Initialize[sbt.Task[UpdateReport]] = Def.taskDyn { - - def grouped[K, V](map: Seq[(K, V)])(mapKey: K => K): Map[K, Seq[V]] = - map.groupBy { case (k, _) => mapKey(k) }.map { - case (k, l) => - k -> l.map { case (_, v) => v } - } - - val so = Organization(scalaOrganization.value) - val internalSbtScalaProvider = appConfiguration.value.provider.scalaProvider - val sbtBootJarOverrides = SbtBootJars( - so, // this seems plain wrong - this assumes that the scala org of the project is the same - // as the one that started SBT. This will scrap the scala org specific JARs by the ones - // that booted SBT, even if the latter come from the standard org.scala-lang org. - // But SBT itself does it this way, and not doing so may make two different versions - // of the scala JARs land in the classpath... - internalSbtScalaProvider.version(), - internalSbtScalaProvider.jars() - ) - - val sv = scalaVersion.value - val sbv = scalaBinaryVersion.value - - val currentProjectTask = - if (sbtClassifiers) - Def.task(FromSbt.sbtClassifiersProject(coursierSbtClassifiersModule.value, sv, sbv)) - else - Def.task { - val proj = coursierProject.value - val publications = coursierPublications.value - - proj.copy(publications = publications) - } - - val log = streams.value.log - - val verbosityLevel = coursierVerbosity.value - - val resTask = - if (withClassifiers && sbtClassifiers) - Def.task { - val cm = coursierSbtClassifiersModule.value - val classifiersRes = coursierSbtClassifiersResolution.value - Map(cm.configurations.map(c => Configuration(c.name)).toSet -> classifiersRes) - } - else - Def.task(coursierResolutions.value) - - // we should be able to call .value on that one here, its conditions don't originate from other tasks - val artifactFilesOrErrors0Task = - if (withClassifiers) { - if (sbtClassifiers) - Keys.coursierSbtClassifiersArtifacts - else - Keys.coursierClassifiersArtifacts - } else if (includeSignatures) - Keys.coursierSignedArtifacts - else - Keys.coursierArtifacts - - val configsTask: sbt.Def.Initialize[sbt.Task[Map[Configuration, Set[Configuration]]]] = - if (withClassifiers && sbtClassifiers) - Def.task { - val cm = coursierSbtClassifiersModule.value - cm.configurations.map(c => Configuration(c.name) -> Set(Configuration(c.name))).toMap - } - else - Def.task { - val configs0 = coursierConfigurations.value - - shadedConfigOpt.fold(configs0) { - case (baseConfig, shadedConfig) => - val baseConfig0 = Configuration(baseConfig) - (configs0 - shadedConfig) + ( - baseConfig0 -> (configs0.getOrElse(baseConfig0, Set()) - shadedConfig) - ) - } - } - - val classifiersTask: sbt.Def.Initialize[sbt.Task[Option[Seq[Classifier]]]] = - if (withClassifiers) { - if (sbtClassifiers) - Def.task { - val cm = coursierSbtClassifiersModule.value - Some(cm.classifiers.map(Classifier(_))) - } - else - Def.task(Some(transitiveClassifiers.value.map(Classifier(_)))) - } else - Def.task(None) - - def reportTask(currentProject: Project, res: Map[Set[Configuration], Resolution]) = Def.task { - - val artifactFilesOrErrors0 = artifactFilesOrErrors0Task.value - val classifiers = classifiersTask.value - val configs = configsTask.value - - val configResolutions = res.flatMap { - case (configs, r) => - configs.iterator.map((_, r)) - } - - def report = { - - val depsByConfig = grouped(currentProject.dependencies)( - config => - shadedConfigOpt match { - case Some((baseConfig, `config`)) => - Configuration(baseConfig) - case _ => - config - } - ) - - if (verbosityLevel >= 2) { - val finalDeps = dependenciesWithConfig( - configResolutions, - depsByConfig.map { case (k, l) => k -> l.toSet }, - configs - ) - - val projCache = res.values.foldLeft(Map.empty[ModuleVersion, Project])(_ ++ _.projectCache.mapValues(_._2)) - val repr = Print.dependenciesUnknownConfigs(finalDeps.toVector, projCache) - log.info(repr.split('\n').map(" " + _).mkString("\n")) - } - - val artifactFiles = artifactFilesOrErrors0.collect { - case (artifact, Right(file)) => - artifact -> file - } - - val artifactErrors = artifactFilesOrErrors0 - .toVector - .collect { - case (a, Left(err)) if !a.optional || !err.notFound => - a -> err - } - - if (artifactErrors.nonEmpty) { - val error = ResolutionError.DownloadErrors(artifactErrors.map(_._2)) - - if (ignoreArtifactErrors) - log.warn(error.description(verbosityLevel >= 1)) - else - error.throwException() - } - - // can be non empty only if ignoreArtifactErrors is true or some optional artifacts are not found - val erroredArtifacts = artifactFilesOrErrors0.collect { - case (artifact, Left(_)) => - artifact - }.toSet - - ToSbt.updateReport( - depsByConfig, - configResolutions, - configs, - classifiers, - artifactFileOpt( - sbtBootJarOverrides, - artifactFiles, - erroredArtifacts, - log, - _, - _, - _, - _ - ), - log, - includeSignatures = includeSignatures - ) - } - - // let's update only one module at once, for a better output - // Downloads are already parallel, no need to parallelize further anyway - synchronized { - report - } - } - - Def.taskDyn { - - val currentProject = currentProjectTask.value - val res = resTask.value - - val key = ReportCacheKey( - currentProject, - res, - withClassifiers, - sbtClassifiers, - ignoreArtifactErrors - ) - - Option(reportsCache.get(key)) match { - case Some(report) => - Def.task(report) - case None => - reportTask(currentProject, res).map { rep => - reportsCache.put(key, rep) - rep - } - } - } - } - - case class ResolutionResult(configs: Set[Configuration], resolution: Resolution, dependencies: Seq[Dependency]) - - private def coursierResolutionTask( - sbtClassifiers: Boolean = false, - ignoreArtifactErrors: Boolean = false - ): Def.Initialize[sbt.Task[Seq[ResolutionResult]]] = Def.taskDyn { - - val currentProjectTask = - if (sbtClassifiers) - Def.task { - val sv = scalaVersion.value - val sbv = scalaBinaryVersion.value - val cm = coursierSbtClassifiersModule.value - FromSbt.sbtClassifiersProject(cm, sv, sbv) - } - else - Def.task { - val proj = coursierProject.value - val publications = coursierPublications.value - proj.copy(publications = publications) - } - - val config = Configuration(configuration.value.name) - val configs = coursierConfigurations.value - - val includedConfigs = configs.getOrElse(config, Set.empty) + config - - Def.taskDyn { - val currentProject = currentProjectTask.value - - val resolutionsTask = - if (sbtClassifiers) - Def.task { - val classifiersRes = coursierSbtClassifiersResolution.value - Map(currentProject.configurations.keySet -> classifiersRes) - } - else - Def.task(coursierResolutions.value) - - Def.task { - val resolutions = resolutionsTask.value - - for { - (subGraphConfigs, res) <- resolutions.toSeq - if subGraphConfigs.exists(includedConfigs) - } yield { - - val dependencies0 = currentProject.dependencies.collect { - case (cfg, dep) if includedConfigs(cfg) && subGraphConfigs(cfg) => dep - }.sortBy { dep => - (dep.module.organization, dep.module.name, dep.version) - } - - val subRes = res.subset(dependencies0.toSet) - - ResolutionResult(subGraphConfigs, subRes, dependencies0) - } - } - } - } - - def coursierDependencyTreeTask( - inverse: Boolean, - sbtClassifiers: Boolean = false, - ignoreArtifactErrors: Boolean = false - ) = Def.task { - val projectName = thisProjectRef.value.project - - val resolutions = coursierResolutionTask(sbtClassifiers, ignoreArtifactErrors).value - for (ResolutionResult(subGraphConfigs, resolution, dependencies) <- resolutions) { - // use sbt logging? - println( - s"$projectName (configurations ${subGraphConfigs.toVector.sorted.mkString(", ")})" + "\n" + - Print.dependencyTree( - dependencies, - resolution, - printExclusions = true, - inverse, - colors = !sys.props.get("sbt.log.noformat").toSeq.contains("true") - ) - ) - } - } - - - def coursierWhatDependsOnTask( - moduleName: String, - sbtClassifiers: Boolean = false, - ignoreArtifactErrors: Boolean = false - ) = Def.task { - val module = Parse.module(moduleName, scalaVersion.value) - .right - .getOrElse(throw new RuntimeException(s"Could not parse module `$moduleName`")) - - val projectName = thisProjectRef.value.project - - val resolutions = coursierResolutionTask(sbtClassifiers, ignoreArtifactErrors).value - val result = new mutable.StringBuilder() - for (ResolutionResult(subGraphConfigs, resolution, _) <- resolutions) { - val roots: Seq[Dependency] = resolution.transitiveDependencies.filter(f => f.module == module) - val strToPrint = s"$projectName (configurations ${subGraphConfigs.toVector.sorted.map(_.value).mkString(", ")})" + "\n" + - Print.reverseTree(roots, resolution, withExclusions = true) - .render(_.repr(Colors.get(!sys.props.get("sbt.log.noformat").toSeq.contains("true")))); - println(strToPrint) - result.append(strToPrint) - result.append("\n") - } - - result.toString - } - -} diff --git a/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsTasks.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsTasks.scala new file mode 100644 index 000000000..7b190dbf0 --- /dev/null +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ArtifactsTasks.scala @@ -0,0 +1,215 @@ +package coursier.sbtcoursier + +import java.io.File + +import coursier.{Artifact, FileError} +import coursier.core._ +import coursier.sbtcoursier.Keys._ +import coursier.sbtcoursier.Structure._ +import sbt.librarymanagement.{Artifact => _, Configuration => _, _} +import sbt.Def +import sbt.Keys._ + +object ArtifactsTasks { + + def coursierPublicationsTask( + configsMap: (sbt.librarymanagement.Configuration, Configuration)* + ): Def.Initialize[sbt.Task[Seq[(Configuration, Publication)]]] = + Def.task { + + val state = sbt.Keys.state.value + val projectRef = sbt.Keys.thisProjectRef.value + val projId = sbt.Keys.projectID.value + val sv = sbt.Keys.scalaVersion.value + val sbv = sbt.Keys.scalaBinaryVersion.value + val ivyConfs = sbt.Keys.ivyConfigurations.value + + val sourcesConfigOpt = + if (ivyConfigurations.value.exists(_.name == "sources")) + Some(Configuration("sources")) + else + None + + val docsConfigOpt = + if (ivyConfigurations.value.exists(_.name == "docs")) + Some(Configuration("docs")) + else + None + + val sbtBinArtifacts = + for ((config, targetConfig) <- configsMap) yield { + + val publish = publishArtifact + .in(projectRef) + .in(packageBin) + .in(config) + .getOrElse(state, false) + + if (publish) + artifact + .in(projectRef) + .in(packageBin) + .in(config) + .find(state) + .map(targetConfig -> _) + else + None + } + + val sbtSourceArtifacts = + for ((config, targetConfig) <- configsMap) yield { + + val publish = publishArtifact + .in(projectRef) + .in(packageSrc) + .in(config) + .getOrElse(state, false) + + if (publish) + artifact + .in(projectRef) + .in(packageSrc) + .in(config) + .find(state) + .map(sourcesConfigOpt.getOrElse(targetConfig) -> _) + else + None + } + + val sbtDocArtifacts = + for ((config, targetConfig) <- configsMap) yield { + + val publish = publishArtifact + .in(projectRef) + .in(packageDoc) + .in(config) + .getOrElse(state, false) + + if (publish) + artifact + .in(projectRef) + .in(packageDoc) + .in(config) + .find(state) + .map(docsConfigOpt.getOrElse(targetConfig) -> _) + else + None + } + + val sbtArtifacts = sbtBinArtifacts ++ sbtSourceArtifacts ++ sbtDocArtifacts + + def artifactPublication(artifact: sbt.Artifact) = { + + val name = FromSbt.sbtCrossVersionName( + artifact.name, + projId.crossVersion, + sv, + sbv + ) + + Publication( + name, + Type(artifact.`type`), + Extension(artifact.extension), + artifact.classifier.fold(Classifier.empty)(Classifier(_)) + ) + } + + val sbtArtifactsPublication = sbtArtifacts.collect { + case Some((config, artifact)) => + config -> artifactPublication(artifact) + } + + val stdArtifactsSet = sbtArtifacts.flatMap(_.map { case (_, a) => a }.toSeq).toSet + + // Second-way of getting artifacts from SBT + // No obvious way of getting the corresponding publishArtifact value for the ones + // only here, it seems. + val extraSbtArtifacts = sbt.Keys.artifacts.in(projectRef).getOrElse(state, Nil) + .filterNot(stdArtifactsSet) + + // Seems that SBT does that - if an artifact has no configs, + // it puts it in all of them. See for example what happens to + // the standalone JAR artifact of the coursier cli module. + def allConfigsIfEmpty(configs: Iterable[ConfigRef]): Iterable[ConfigRef] = + if (configs.isEmpty) ivyConfs.filter(_.isPublic).map(c => ConfigRef(c.name)) else configs + + val extraSbtArtifactsPublication = for { + artifact <- extraSbtArtifacts + config <- allConfigsIfEmpty(artifact.configurations.map(x => ConfigRef(x.name))) + // FIXME If some configurations from artifact.configurations are not public, they may leak here :\ + } yield Configuration(config.name) -> artifactPublication(artifact) + + sbtArtifactsPublication ++ extraSbtArtifactsPublication + } + + def artifactsTask( + withClassifiers: Boolean, + sbtClassifiers: Boolean = false, + ignoreArtifactErrors: Boolean = false, + includeSignatures: Boolean = false + ): Def.Initialize[sbt.Task[Map[Artifact, Either[FileError, File]]]] = { + + val resTask: sbt.Def.Initialize[sbt.Task[Seq[Resolution]]] = + if (withClassifiers && sbtClassifiers) + Def.task(Seq(coursierSbtClassifiersResolution.value)) + else + Def.task(coursierResolutions.value.values.toVector) + + val classifiersTask: sbt.Def.Initialize[sbt.Task[Option[Seq[Classifier]]]] = + if (withClassifiers) { + if (sbtClassifiers) + Def.task(Some(coursierSbtClassifiersModule.value.classifiers.map(Classifier(_)))) + else + Def.task(Some(transitiveClassifiers.value.map(Classifier(_)))) + } else + Def.task(None) + + Def.task { + + val projectName = thisProjectRef.value.project + + val parallelDownloads = coursierParallelDownloads.value + val artifactsChecksums = coursierArtifactsChecksums.value + val cachePolicies = coursierCachePolicies.value + val ttl = coursierTtl.value + val cache = coursierCache.value + val createLogger = coursierCreateLogger.value + + val log = streams.value.log + + val verbosityLevel = coursierVerbosity.value + + val classifiers = classifiersTask.value + val res = resTask.value + + val params = ArtifactsParams( + classifiers, + res, + includeSignatures, + parallelDownloads, + createLogger, + cache, + artifactsChecksums, + ttl, + cachePolicies, + projectName, + sbtClassifiers + ) + + val resOrError = ArtifactsRun.artifacts( + params, + verbosityLevel, + log + ) + + resOrError match { + case Left(err) => + err.throwException() + case Right(res) => + res + } + } + } + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/CoursierPlugin.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/CoursierPlugin.scala similarity index 86% rename from modules/sbt-coursier/src/main/scala/coursier/CoursierPlugin.scala rename to modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/CoursierPlugin.scala index d55cc7ebb..a413561ad 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/CoursierPlugin.scala +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/CoursierPlugin.scala @@ -1,10 +1,11 @@ -package coursier +package coursier.sbtcoursier import java.io.OutputStreamWriter -import coursier.core.Configuration +import coursier.{Cache, CachePolicy, TermDisplay} +import coursier.core.{Configuration, ResolutionProcess} import sbt.librarymanagement.{Configuration => _, Resolver => _, _} -import sbt.{Configuration => _, _} +import sbt.{Cache => _, Configuration => _, _} import sbt.Keys._ object CoursierPlugin extends AutoPlugin { @@ -64,16 +65,16 @@ object CoursierPlugin extends AutoPlugin { import autoImport._ lazy val treeSettings = Seq( - coursierDependencyTree := Tasks.coursierDependencyTreeTask( + coursierDependencyTree := DisplayTasks.coursierDependencyTreeTask( inverse = false ).value, - coursierDependencyInverseTree := Tasks.coursierDependencyTreeTask( + coursierDependencyInverseTree := DisplayTasks.coursierDependencyTreeTask( inverse = true ).value, coursierWhatDependsOn := Def.inputTaskDyn { import sbt.complete.DefaultParsers._ val input = token(SpaceClass ~ NotQuoted, "").parsed._2 - Tasks.coursierWhatDependsOnTask(input) + DisplayTasks.coursierWhatDependsOnTask(input) }.evaluated ) @@ -165,11 +166,10 @@ object CoursierPlugin extends AutoPlugin { ) = hackHack ++ Seq( clean := { val noWarningPlz = clean.value - Tasks.resolutionsCache.clear() - Tasks.reportsCache.clear() + SbtCoursierCache.default.clear() }, - coursierResolvers := Tasks.coursierResolversTask.value, - coursierRecursiveResolvers := Tasks.coursierRecursiveResolversTask.value, + coursierResolvers := RepositoriesTasks.coursierResolversTask.value, + coursierRecursiveResolvers := RepositoriesTasks.coursierRecursiveResolversTask.value, coursierSbtResolvers := { // TODO Add docker-based integration test for that, see https://github.com/coursier/coursier/issues/632 @@ -201,39 +201,39 @@ object CoursierPlugin extends AutoPlugin { !r.name.startsWith("local-preloaded") } }, - coursierFallbackDependencies := Tasks.coursierFallbackDependenciesTask.value, - coursierArtifacts := Tasks.artifactFilesOrErrors(withClassifiers = false).value, - coursierSignedArtifacts := Tasks.artifactFilesOrErrors(withClassifiers = false, includeSignatures = true).value, - coursierClassifiersArtifacts := Tasks.artifactFilesOrErrors( + coursierFallbackDependencies := InputsTasks.coursierFallbackDependenciesTask.value, + coursierArtifacts := ArtifactsTasks.artifactsTask(withClassifiers = false).value, + coursierSignedArtifacts := ArtifactsTasks.artifactsTask(withClassifiers = false, includeSignatures = true).value, + coursierClassifiersArtifacts := ArtifactsTasks.artifactsTask( withClassifiers = true ).value, - coursierSbtClassifiersArtifacts := Tasks.artifactFilesOrErrors( + coursierSbtClassifiersArtifacts := ArtifactsTasks.artifactsTask( withClassifiers = true, sbtClassifiers = true ).value, - update := Tasks.updateTask( + update := UpdateTasks.updateTask( shadedConfigOpt, withClassifiers = false ).value, - updateClassifiers := Tasks.updateTask( + updateClassifiers := UpdateTasks.updateTask( shadedConfigOpt, withClassifiers = true, ignoreArtifactErrors = true ).value, - updateSbtClassifiers.in(Defaults.TaskGlobal) := Tasks.updateTask( + updateSbtClassifiers.in(Defaults.TaskGlobal) := UpdateTasks.updateTask( shadedConfigOpt, withClassifiers = true, sbtClassifiers = true, ignoreArtifactErrors = true ).value, - coursierProject := Tasks.coursierProjectTask.value, - coursierConfigGraphs := Tasks.ivyGraphsTask.value, - coursierInterProjectDependencies := Tasks.coursierInterProjectDependenciesTask.value, - coursierPublications := Tasks.coursierPublicationsTask(packageConfigs: _*).value, + coursierProject := InputsTasks.coursierProjectTask.value, + coursierConfigGraphs := InputsTasks.ivyGraphsTask.value, + coursierInterProjectDependencies := InputsTasks.coursierInterProjectDependenciesTask.value, + coursierPublications := ArtifactsTasks.coursierPublicationsTask(packageConfigs: _*).value, coursierSbtClassifiersModule := classifiersModule.in(updateSbtClassifiers).value, - coursierConfigurations := Tasks.coursierConfigurationsTask(None).value, - coursierParentProjectCache := Tasks.parentProjectCacheTask.value, - coursierResolutions := Tasks.resolutionsTask().value, + coursierConfigurations := InputsTasks.coursierConfigurationsTask(None).value, + coursierParentProjectCache := InputsTasks.parentProjectCacheTask.value, + coursierResolutions := ResolutionTasks.resolutionsTask().value, Keys.actualCoursierResolution := { val config = Configuration(Compile.name) @@ -248,7 +248,7 @@ object CoursierPlugin extends AutoPlugin { sys.error(s"Resolution for configuration $config not found") } }, - coursierSbtClassifiersResolution := Tasks.resolutionsTask( + coursierSbtClassifiersResolution := ResolutionTasks.resolutionsTask( sbtClassifiers = true ).value.head._2, ivyConfigurations := { diff --git a/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/DisplayTasks.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/DisplayTasks.scala new file mode 100644 index 000000000..ac4d23c95 --- /dev/null +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/DisplayTasks.scala @@ -0,0 +1,128 @@ +package coursier.sbtcoursier + +import coursier.core._ +import coursier.sbtcoursier.Keys._ +import coursier.util.Print.Colors +import coursier.util.{Parse, Print} +import sbt.Def +import sbt.Keys._ + +import scala.collection.mutable + +object DisplayTasks { + + private case class ResolutionResult(configs: Set[Configuration], resolution: Resolution, dependencies: Seq[Dependency]) + + private def coursierResolutionTask( + sbtClassifiers: Boolean = false, + ignoreArtifactErrors: Boolean = false + ): Def.Initialize[sbt.Task[Seq[ResolutionResult]]] = { + + val currentProjectTask = + if (sbtClassifiers) + Def.task { + val sv = scalaVersion.value + val sbv = scalaBinaryVersion.value + val cm = coursierSbtClassifiersModule.value + FromSbt.sbtClassifiersProject(cm, sv, sbv) + } + else + Def.task { + val proj = coursierProject.value + val publications = coursierPublications.value + proj.copy(publications = publications) + } + + Def.taskDyn { + + val config = Configuration(configuration.value.name) + val configs = coursierConfigurations.value + + val includedConfigs = configs.getOrElse(config, Set.empty) + config + + Def.taskDyn { + val currentProject = currentProjectTask.value + + val resolutionsTask = + if (sbtClassifiers) + Def.task { + val classifiersRes = coursierSbtClassifiersResolution.value + Map(currentProject.configurations.keySet -> classifiersRes) + } + else + Def.task(coursierResolutions.value) + + Def.task { + val resolutions = resolutionsTask.value + + for { + (subGraphConfigs, res) <- resolutions.toSeq + if subGraphConfigs.exists(includedConfigs) + } yield { + + val dependencies0 = currentProject.dependencies.collect { + case (cfg, dep) if includedConfigs(cfg) && subGraphConfigs(cfg) => dep + }.sortBy { dep => + (dep.module.organization, dep.module.name, dep.version) + } + + val subRes = res.subset(dependencies0.toSet) + + ResolutionResult(subGraphConfigs, subRes, dependencies0) + } + } + } + } + } + + def coursierDependencyTreeTask( + inverse: Boolean, + sbtClassifiers: Boolean = false, + ignoreArtifactErrors: Boolean = false + ) = Def.task { + val projectName = thisProjectRef.value.project + + val resolutions = coursierResolutionTask(sbtClassifiers, ignoreArtifactErrors).value + for (ResolutionResult(subGraphConfigs, resolution, dependencies) <- resolutions) { + // use sbt logging? + println( + s"$projectName (configurations ${subGraphConfigs.toVector.sorted.mkString(", ")})" + "\n" + + Print.dependencyTree( + dependencies, + resolution, + printExclusions = true, + inverse, + colors = !sys.props.get("sbt.log.noformat").toSeq.contains("true") + ) + ) + } + } + + + def coursierWhatDependsOnTask( + moduleName: String, + sbtClassifiers: Boolean = false, + ignoreArtifactErrors: Boolean = false + ) = Def.task { + val module = Parse.module(moduleName, scalaVersion.value) + .right + .getOrElse(throw new RuntimeException(s"Could not parse module `$moduleName`")) + + val projectName = thisProjectRef.value.project + + val resolutions = coursierResolutionTask(sbtClassifiers, ignoreArtifactErrors).value + val result = new mutable.StringBuilder + for (ResolutionResult(subGraphConfigs, resolution, _) <- resolutions) { + val roots: Seq[Dependency] = resolution.transitiveDependencies.filter(f => f.module == module) + val strToPrint = s"$projectName (configurations ${subGraphConfigs.toVector.sorted.map(_.value).mkString(", ")})" + "\n" + + Print.reverseTree(roots, resolution, withExclusions = true) + .render(_.repr(Colors.get(!sys.props.get("sbt.log.noformat").toSeq.contains("true")))) + println(strToPrint) + result.append(strToPrint) + result.append("\n") + } + + result.toString + } + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/InputsTasks.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/InputsTasks.scala new file mode 100644 index 000000000..c75fb9847 --- /dev/null +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/InputsTasks.scala @@ -0,0 +1,122 @@ +package coursier.sbtcoursier + +import java.net.URL + +import coursier.ProjectCache +import coursier.core._ +import coursier.sbtcoursier.Keys._ +import coursier.sbtcoursier.Structure._ +import sbt.librarymanagement.{Configuration => _, _} +import sbt.Def +import sbt.Keys._ + +object InputsTasks { + + def coursierFallbackDependenciesTask: Def.Initialize[sbt.Task[Seq[(Module, String, URL, Boolean)]]] = + Def.taskDyn { + + val state = sbt.Keys.state.value + val projectRef = sbt.Keys.thisProjectRef.value + + val projects = Structure.allRecursiveInterDependencies(state, projectRef) + + val allDependenciesTask = allDependencies + .forAllProjects(state, projectRef +: projects) + .map(_.values.toVector.flatten) + + Def.task { + val allDependencies = allDependenciesTask.value + + FromSbt.fallbackDependencies( + allDependencies, + scalaVersion.in(projectRef).get(state), + scalaBinaryVersion.in(projectRef).get(state) + ) + } + } + + def coursierProjectTask: Def.Initialize[sbt.Task[Project]] = + Def.taskDyn { + + val state = sbt.Keys.state.value + val projectRef = sbt.Keys.thisProjectRef.value + + val allDependenciesTask = allDependencies.in(projectRef).get(state) + + Def.task { + Inputs.coursierProject( + projectID.in(projectRef).get(state), + allDependenciesTask.value, + excludeDependencies.in(projectRef).get(state), + // should projectID.configurations be used instead? + ivyConfigurations.in(projectRef).get(state), + scalaVersion.in(projectRef).get(state), + scalaBinaryVersion.in(projectRef).get(state), + state.log + ) + } + } + + def coursierInterProjectDependenciesTask: Def.Initialize[sbt.Task[Seq[Project]]] = + Def.taskDyn { + + val state = sbt.Keys.state.value + val projectRef = sbt.Keys.thisProjectRef.value + + val projects = Structure.allRecursiveInterDependencies(state, projectRef) + + val t = coursierProject.forAllProjects(state, projects).map(_.values.toVector) + + Def.task(t.value) + } + + def coursierConfigurationsTask( + shadedConfig: Option[(String, Configuration)] + ): Def.Initialize[sbt.Task[Map[Configuration, Set[Configuration]]]] = + Def.task { + Inputs.coursierConfigurations(ivyConfigurations.value, shadedConfig) + } + + def ivyGraphsTask: Def.Initialize[sbt.Task[Seq[Set[Configuration]]]] = + Def.task { + val p = coursierProject.value + Inputs.ivyGraphs(p.configurations) + } + + def parentProjectCacheTask: Def.Initialize[sbt.Task[Map[Seq[sbt.librarymanagement.Resolver], Seq[coursier.ProjectCache]]]] = + Def.taskDyn { + + val state = sbt.Keys.state.value + val projectRef = sbt.Keys.thisProjectRef.value + + val projectDeps = structure(state).allProjects + .find(_.id == projectRef.project) + .map(_.dependencies.map(_.project.project).toSet) + .getOrElse(Set.empty) + + val projects = structure(state).allProjectRefs.filter(p => projectDeps(p.project)) + + val t = + for { + m <- coursierRecursiveResolvers.forAllProjects(state, projects) + n <- coursierResolutions.forAllProjects(state, m.keys.toSeq) + } yield + n.foldLeft(Map.empty[Seq[Resolver], Seq[ProjectCache]]) { + case (caches, (ref, resolutions)) => + val mainResOpt = resolutions.collectFirst { + case (k, v) if k(Configuration.compile) => v + } + + val r = for { + resolvers <- m.get(ref) + resolution <- mainResOpt + } yield + caches.updated(resolvers, resolution.projectCache +: caches.getOrElse(resolvers, Seq.empty)) + + r.getOrElse(caches) + } + + Def.task(t.value) + } + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/IvyXml.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/IvyXml.scala similarity index 98% rename from modules/sbt-coursier/src/main/scala/coursier/IvyXml.scala rename to modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/IvyXml.scala index 436d32b44..9b66ef972 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/IvyXml.scala +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/IvyXml.scala @@ -1,9 +1,9 @@ -package coursier +package coursier.sbtcoursier import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.Files -import coursier.core.Configuration +import coursier.core.{Configuration, Project} import org.apache.ivy.core.module.id.ModuleRevisionId import scala.collection.JavaConverters._ diff --git a/modules/sbt-coursier/src/main/scala/coursier/Keys.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Keys.scala similarity index 94% rename from modules/sbt-coursier/src/main/scala/coursier/Keys.scala rename to modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Keys.scala index 3e024d470..dbc81a5f8 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/Keys.scala +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Keys.scala @@ -1,11 +1,12 @@ -package coursier +package coursier.sbtcoursier import java.io.File import java.net.URL -import coursier.core.{Configuration, Publication} -import sbt.librarymanagement.GetClassifiersModule -import sbt.{InputKey, Resolver, SettingKey, TaskKey} +import coursier.{Cache, CachePolicy, Credentials, FileError, ProjectCache} +import coursier.core._ +import sbt.librarymanagement.{GetClassifiersModule, Resolver} +import sbt.{InputKey, SettingKey, TaskKey} import scala.concurrent.duration.Duration diff --git a/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/RepositoriesTasks.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/RepositoriesTasks.scala new file mode 100644 index 000000000..7b6e375a8 --- /dev/null +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/RepositoriesTasks.scala @@ -0,0 +1,71 @@ +package coursier.sbtcoursier + +import coursier.sbtcoursier.Keys._ +import coursier.sbtcoursier.Structure._ +import sbt.{Classpaths, Def} +import sbt.Keys._ +import sbt.librarymanagement.Resolver + +object RepositoriesTasks { + + private def resultTask(bootResOpt: Option[Seq[Resolver]], overrideFlag: Boolean): Def.Initialize[sbt.Task[Seq[Resolver]]] = + bootResOpt.filter(_ => overrideFlag) match { + case Some(r) => Def.task(r) + case None => + Def.taskDyn { + val extRes = externalResolvers.value + val isSbtPlugin = sbtPlugin.value + if (isSbtPlugin) + Def.task { + Seq( + sbtResolver.value, + Classpaths.sbtPluginReleases + ) ++ extRes + } + else + Def.task(extRes) + } + } + + def coursierResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = + Def.taskDyn { + + val bootResOpt = bootResolvers.value + val overrideFlag = overrideBuildResolvers.value + + Def.task { + val result = resultTask(bootResOpt, overrideFlag).value + val reorderResolvers = coursierReorderResolvers.value + val keepPreloaded = coursierKeepPreloaded.value + + val result0 = + if (reorderResolvers) + ResolutionParams.reorderResolvers(result) + else + result + + if (keepPreloaded) + result0 + else + result0.filter { r => + !r.name.startsWith("local-preloaded") + } + } + } + + def coursierRecursiveResolversTask: Def.Initialize[sbt.Task[Seq[Resolver]]] = + Def.taskDyn { + + val state = sbt.Keys.state.value + val projectRef = sbt.Keys.thisProjectRef.value + + val projects = Structure.allRecursiveInterDependencies(state, projectRef) + + val t = coursierResolvers + .forAllProjects(state, projectRef +: projects) + .map(_.values.toVector.flatten) + + Def.task(t.value) + } + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ResolutionTasks.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ResolutionTasks.scala new file mode 100644 index 000000000..0c2805989 --- /dev/null +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ResolutionTasks.scala @@ -0,0 +1,224 @@ +package coursier.sbtcoursier + +import java.net.URL + +import coursier.{Cache, ProjectCache} +import coursier.core._ +import coursier.extra.Typelevel +import coursier.ivy.IvyRepository +import coursier.maven.MavenRepository +import coursier.sbtcoursier.Keys._ +import sbt.Def +import sbt.Keys._ + +import scala.util.Try + +object ResolutionTasks { + + def resolutionsTask( + sbtClassifiers: Boolean = false + ): Def.Initialize[sbt.Task[Map[Set[Configuration], coursier.Resolution]]] = { + + val currentProjectTask: sbt.Def.Initialize[sbt.Task[(Project, Seq[(Module, String, URL, Boolean)], Seq[Set[Configuration]])]] = + if (sbtClassifiers) + Def.task { + val sv = scalaVersion.value + val sbv = scalaBinaryVersion.value + val cm = coursierSbtClassifiersModule.value + val proj = FromSbt.sbtClassifiersProject(cm, sv, sbv) + + val fallbackDeps = FromSbt.fallbackDependencies( + cm.dependencies, + sv, + sbv + ) + + (proj, fallbackDeps, Vector(cm.configurations.map(c => Configuration(c.name)).toSet)) + } + else + Def.task { + val baseConfigGraphs = coursierConfigGraphs.value + (coursierProject.value.copy(publications = coursierPublications.value), coursierFallbackDependencies.value, baseConfigGraphs) + } + + val resolversTask = + if (sbtClassifiers) + Def.task(coursierSbtResolvers.value) + else + Def.task(coursierRecursiveResolvers.value.distinct) + + val authenticationByHostTask = Def.taskDyn { + + val useSbtCredentials = coursierUseSbtCredentials.value + + if (useSbtCredentials) + Def.task { + val log = streams.value.log + + sbt.Keys.credentials.value + .flatMap { + case dc: sbt.DirectCredentials => List(dc) + case fc: sbt.FileCredentials => + sbt.Credentials.loadCredentials(fc.path) match { + case Left(err) => + log.warn(s"$err, ignoring it") + Nil + case Right(dc) => List(dc) + } + } + .map { c => + c.host -> Authentication(c.userName, c.passwd) + } + .toMap + } + else + Def.task(Map.empty[String, Authentication]) + } + + Def.task { + val projectName = thisProjectRef.value.project + + val sv = scalaVersion.value + val sbv = scalaBinaryVersion.value + + val interProjectDependencies = coursierInterProjectDependencies.value + + val parallelDownloads = coursierParallelDownloads.value + val checksums = coursierChecksums.value + val maxIterations = coursierMaxIterations.value + val cachePolicies = coursierCachePolicies.value + val ttl = coursierTtl.value + val cache = coursierCache.value + val createLogger = coursierCreateLogger.value + + val log = streams.value.log + + // are these always defined? (e.g. for Java only projects?) + val so = Organization(scalaOrganization.value) + + val userForceVersions = dependencyOverrides + .value + .map(FromSbt.moduleVersion(_, sv, sbv)) + .toMap + + val verbosityLevel = coursierVerbosity.value + + val userEnabledProfiles = mavenProfiles.value + + val typelevel = Organization(scalaOrganization.value) == Typelevel.typelevelOrg + + val globalPluginsRepos = + for (p <- ResolutionParams.globalPluginPatterns(sbtBinaryVersion.value)) + yield IvyRepository.fromPattern( + p, + withChecksums = false, + withSignatures = false, + withArtifacts = false + ) + + val interProjectRepo = InterProjectRepository(interProjectDependencies) + + val ivyProperties = ResolutionParams.defaultIvyProperties() + + val authenticationByRepositoryId = coursierCredentials.value.mapValues(_.authentication) + + val (currentProject, fallbackDependencies, configGraphs) = currentProjectTask.value + + val autoScalaLib = autoScalaLibrary.value + + val resolvers = resolversTask.value + + // TODO Warn about possible duplicated modules from source repositories? + + val authenticationByHost = authenticationByHostTask.value + + val internalRepositories = globalPluginsRepos :+ interProjectRepo + + val parentProjectCache: ProjectCache = coursierParentProjectCache.value + .get(resolvers) + .map(_.foldLeft[ProjectCache](Map.empty)(_ ++ _)) + .getOrElse(Map.empty) + + def withAuthenticationByHost(repo: Repository, credentials: Map[String, Authentication]): Repository = { + + def httpHost(s: String) = + if (s.startsWith("http://") || s.startsWith("https://")) + Try(Cache.url(s).getHost).toOption + else + None + + repo match { + case m: MavenRepository => + if (m.authentication.isEmpty) + httpHost(m.root).flatMap(credentials.get).fold(m) { auth => + m.copy(authentication = Some(auth)) + } + else + m + case i: IvyRepository => + if (i.authentication.isEmpty) { + val base = i.pattern.chunks.takeWhile { + case _: coursier.ivy.Pattern.Chunk.Const => true + case _ => false + }.map(_.string).mkString + + httpHost(base).flatMap(credentials.get).fold(i) { auth => + i.copy(authentication = Some(auth)) + } + } else + i + case _ => + repo + } + } + + val mainRepositories = resolvers + .flatMap { resolver => + FromSbt.repository( + resolver, + ivyProperties, + log, + authenticationByRepositoryId.get(resolver.name) + ) + } + .map(withAuthenticationByHost(_, authenticationByHost)) + + val resOrError = ResolutionRun.resolutions( + ResolutionParams( + currentProject.dependencies, + fallbackDependencies, + configGraphs, + autoScalaLib, + mainRepositories, + parentProjectCache, + interProjectDependencies, + internalRepositories, + userEnabledProfiles, + userForceVersions, + typelevel, + so, + sv, + sbtClassifiers, + parallelDownloads, + projectName, + maxIterations, + createLogger, + cache, + cachePolicies, + ttl, + checksums + ), + verbosityLevel, + log + ) + + resOrError match { + case Left(err) => + err.throwException() + case Right(res) => + res + } + } + } + +} diff --git a/modules/sbt-coursier/src/main/scala/coursier/Settings.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Settings.scala similarity index 97% rename from modules/sbt-coursier/src/main/scala/coursier/Settings.scala rename to modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Settings.scala index 61e27223e..fd6abc25c 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/Settings.scala +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Settings.scala @@ -1,4 +1,4 @@ -package coursier +package coursier.sbtcoursier import sbt.Logger diff --git a/modules/sbt-coursier/src/main/scala/coursier/Structure.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Structure.scala similarity index 50% rename from modules/sbt-coursier/src/main/scala/coursier/Structure.scala rename to modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Structure.scala index e8b09bd83..cada0a1a0 100644 --- a/modules/sbt-coursier/src/main/scala/coursier/Structure.scala +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/Structure.scala @@ -1,10 +1,34 @@ -package coursier +package coursier.sbtcoursier import sbt._ -// things from sbt-structure object Structure { + def allRecursiveInterDependencies(state: sbt.State, projectRef: sbt.ProjectRef) = { + + def dependencies(map: Map[String, Seq[String]], id: String): Set[String] = { + + def helper(map: Map[String, Seq[String]], acc: Set[String]): Set[String] = + if (acc.exists(map.contains)) { + val (kept, rem) = map.partition { case (k, _) => acc(k) } + helper(rem, acc ++ kept.valuesIterator.flatten) + } else + acc + + helper(map - id, map.getOrElse(id, Nil).toSet) + } + + val allProjectsDeps = + for (p <- structure(state).allProjects) + yield p.id -> p.dependencies.map(_.project.project) + + val deps = dependencies(allProjectsDeps.toMap, projectRef.project) + + structure(state).allProjectRefs.filter(p => deps(p.project)) + } + + // vv things from sbt-structure vv + def structure(state: State) = sbt.Project.structure(state) diff --git a/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/UpdateTasks.scala b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/UpdateTasks.scala new file mode 100644 index 000000000..64e85fb54 --- /dev/null +++ b/modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/UpdateTasks.scala @@ -0,0 +1,153 @@ +package coursier.sbtcoursier + +import coursier.core._ +import coursier.sbtcoursier.Keys._ +import sbt.Def +import sbt.Keys._ +import sbt.librarymanagement.UpdateReport + +object UpdateTasks { + + def updateTask( + shadedConfigOpt: Option[(String, Configuration)], + withClassifiers: Boolean, + sbtClassifiers: Boolean = false, + ignoreArtifactErrors: Boolean = false, + includeSignatures: Boolean = false + ): Def.Initialize[sbt.Task[UpdateReport]] = { + + val currentProjectTask = + if (sbtClassifiers) + Def.task { + val sv = scalaVersion.value + val sbv = scalaBinaryVersion.value + FromSbt.sbtClassifiersProject(coursierSbtClassifiersModule.value, sv, sbv) + } + else + Def.task { + val proj = coursierProject.value + val publications = coursierPublications.value + + proj.copy(publications = publications) + } + + val resTask = + if (withClassifiers && sbtClassifiers) + Def.task { + val cm = coursierSbtClassifiersModule.value + val classifiersRes = coursierSbtClassifiersResolution.value + Map(cm.configurations.map(c => Configuration(c.name)).toSet -> classifiersRes) + } + else + Def.task(coursierResolutions.value) + + // we should be able to call .value on that one here, its conditions don't originate from other tasks + val artifactFilesOrErrors0Task = + if (withClassifiers) { + if (sbtClassifiers) + Keys.coursierSbtClassifiersArtifacts + else + Keys.coursierClassifiersArtifacts + } else if (includeSignatures) + Keys.coursierSignedArtifacts + else + Keys.coursierArtifacts + + val configsTask: sbt.Def.Initialize[sbt.Task[Map[Configuration, Set[Configuration]]]] = + if (withClassifiers && sbtClassifiers) + Def.task { + val cm = coursierSbtClassifiersModule.value + cm.configurations.map(c => Configuration(c.name) -> Set(Configuration(c.name))).toMap + } + else + Def.task { + val configs0 = coursierConfigurations.value + + shadedConfigOpt.fold(configs0) { + case (baseConfig, shadedConfig) => + val baseConfig0 = Configuration(baseConfig) + (configs0 - shadedConfig) + ( + baseConfig0 -> (configs0.getOrElse(baseConfig0, Set()) - shadedConfig) + ) + } + } + + val classifiersTask: sbt.Def.Initialize[sbt.Task[Option[Seq[Classifier]]]] = + if (withClassifiers) { + if (sbtClassifiers) + Def.task { + val cm = coursierSbtClassifiersModule.value + Some(cm.classifiers.map(Classifier(_))) + } + else + Def.task(Some(transitiveClassifiers.value.map(Classifier(_)))) + } else + Def.task(None) + + Def.taskDyn { + + val so = Organization(scalaOrganization.value) + val internalSbtScalaProvider = appConfiguration.value.provider.scalaProvider + val sbtBootJarOverrides = SbtBootJars( + so, // this seems plain wrong - this assumes that the scala org of the project is the same + // as the one that started SBT. This will scrap the scala org specific JARs by the ones + // that booted SBT, even if the latter come from the standard org.scala-lang org. + // But SBT itself does it this way, and not doing so may make two different versions + // of the scala JARs land in the classpath... + internalSbtScalaProvider.version(), + internalSbtScalaProvider.jars() + ) + + val log = streams.value.log + + val verbosityLevel = coursierVerbosity.value + + val dependencies = currentProjectTask.value.dependencies + val res = resTask.value + + val key = SbtCoursierCache.ReportKey( + dependencies, + res, + withClassifiers, + sbtClassifiers, + ignoreArtifactErrors + ) + + SbtCoursierCache.default.reportOpt(key) match { + case Some(report) => + Def.task(report) + case None => + Def.task { + + val artifactFilesOrErrors0 = artifactFilesOrErrors0Task.value + val classifiers = classifiersTask.value + val configs = configsTask.value + + val params = UpdateParams( + shadedConfigOpt, + artifactFilesOrErrors0, + classifiers, + configs, + dependencies, + res, + ignoreArtifactErrors, + includeSignatures, + sbtBootJarOverrides + ) + + val repOrError = + Lock.lock.synchronized { + UpdateRun.update(params, verbosityLevel, log) + } + for (rep <- repOrError) + SbtCoursierCache.default.putReport(key, rep) + repOrError match { + case Left(err) => err.throwException() + case Right(rep) => rep + } + } + } + } + } + +} diff --git a/modules/sbt-coursier/src/sbt-test/sbt-coursier-group-1/clean/project/helper.scala b/modules/sbt-coursier/src/sbt-test/sbt-coursier-group-1/clean/project/helper.scala index 000111d37..5e57a26bc 100644 --- a/modules/sbt-coursier/src/sbt-test/sbt-coursier-group-1/clean/project/helper.scala +++ b/modules/sbt-coursier/src/sbt-test/sbt-coursier-group-1/clean/project/helper.scala @@ -2,8 +2,7 @@ package coursier object Helper { - def checkEmpty(): Boolean = { - Tasks.resolutionsCache.isEmpty && Tasks.reportsCache.isEmpty - } + def checkEmpty(): Boolean = + coursier.sbtcoursier.SbtCoursierCache.default.isEmpty } \ No newline at end of file diff --git a/modules/sbt-coursier/src/test/scala/coursier/IvyXmlTests.scala b/modules/sbt-coursier/src/test/scala/coursier/sbtcoursier/IvyXmlTests.scala similarity index 86% rename from modules/sbt-coursier/src/test/scala/coursier/IvyXmlTests.scala rename to modules/sbt-coursier/src/test/scala/coursier/sbtcoursier/IvyXmlTests.scala index 74f911635..91d8ce783 100644 --- a/modules/sbt-coursier/src/test/scala/coursier/IvyXmlTests.scala +++ b/modules/sbt-coursier/src/test/scala/coursier/sbtcoursier/IvyXmlTests.scala @@ -1,6 +1,7 @@ -package coursier +package coursier.sbtcoursier import coursier.core.Configuration +import coursier.{Info, Module, Project, moduleNameString, organizationString} import utest._ object IvyXmlTests extends TestSuite { diff --git a/modules/sbt-pgp-coursier/src/main/scala/coursier/CoursierSbtPgpPlugin.scala b/modules/sbt-pgp-coursier/src/main/scala/coursier/CoursierSbtPgpPlugin.scala index c5fa119d0..6dd1e80ce 100644 --- a/modules/sbt-pgp-coursier/src/main/scala/coursier/CoursierSbtPgpPlugin.scala +++ b/modules/sbt-pgp-coursier/src/main/scala/coursier/CoursierSbtPgpPlugin.scala @@ -1,17 +1,18 @@ package coursier import com.typesafe.sbt.pgp.PgpKeys.updatePgpSignatures +import coursier.sbtcoursier.UpdateTasks import sbt.AutoPlugin object CoursierSbtPgpPlugin extends AutoPlugin { override def trigger = allRequirements - override def requires = com.typesafe.sbt.SbtPgp && coursier.CoursierPlugin + override def requires = com.typesafe.sbt.SbtPgp && coursier.sbtcoursier.CoursierPlugin override val projectSettings = Seq( updatePgpSignatures := { - Tasks.updateTask( + UpdateTasks.updateTask( None, withClassifiers = false, includeSignatures = true diff --git a/modules/sbt-shading/src/main/scala/coursier/ShadingPlugin.scala b/modules/sbt-shading/src/main/scala/coursier/ShadingPlugin.scala index 88d6875cc..e849ab80b 100644 --- a/modules/sbt-shading/src/main/scala/coursier/ShadingPlugin.scala +++ b/modules/sbt-shading/src/main/scala/coursier/ShadingPlugin.scala @@ -4,6 +4,7 @@ import java.io.File import coursier.core.{Configuration, Type} import coursier.ivy.IvyXml.{mappings => ivyXmlMappings} +import coursier.sbtcoursier.{CoursierPlugin, InputsTasks, Keys} import sbt.librarymanagement._ import sbt.Keys._ import sbt.{AutoPlugin, Compile, SettingKey, TaskKey, inConfig} @@ -71,7 +72,7 @@ object ShadingPlugin extends AutoPlugin { packagedArtifacts := sbt.Classpaths.packaged(shadingDefaultArtifactTasks).value ) - import CoursierPlugin.autoImport._ + import coursier.sbtcoursier.CoursierPlugin.autoImport._ override lazy val buildSettings = super.buildSettings ++ Seq( shadeNamespaces := Set() @@ -79,7 +80,7 @@ object ShadingPlugin extends AutoPlugin { override lazy val projectSettings = Seq( - coursierConfigurations := Tasks.coursierConfigurationsTask( + coursierConfigurations := InputsTasks.coursierConfigurationsTask( Some(baseDependencyConfiguration.value -> Configuration(Shaded.name)) ).value, ivyConfigurations := Shaded +: ivyConfigurations.value.map {