Merge pull request #96 from scalacenter/sbt-chain-resolver-rewrite

Make `SbtChainResolver` readable
This commit is contained in:
eugene yokota 2017-05-12 12:40:26 -04:00 committed by GitHub
commit 40fa6dcd3b
1 changed files with 223 additions and 170 deletions

View File

@ -1,21 +1,20 @@
package sbt.internal.librarymanagement package sbt.internal.librarymanagement
package ivyint package ivyint
import java.io.File import java.io.{ ByteArrayOutputStream, File, PrintWriter }
import java.text.ParseException import java.text.ParseException
import java.util.Date import java.util.Date
import org.apache.ivy.core.cache.ArtifactOrigin
import org.apache.ivy.core.settings.IvySettings import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.core.{ IvyContext, LogOptions } import org.apache.ivy.core.{ IvyContext, LogOptions }
import org.apache.ivy.core.module.descriptor.{ import org.apache.ivy.core.module.descriptor.DefaultModuleDescriptor
Artifact => IArtifact, import org.apache.ivy.core.module.descriptor.DependencyDescriptor
DefaultModuleDescriptor, import org.apache.ivy.core.module.descriptor.ModuleDescriptor
ModuleDescriptor, import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact }
DependencyDescriptor import org.apache.ivy.core.resolve.{ ResolveData, ResolvedModuleRevision }
}
import org.apache.ivy.core.resolve.{ ResolvedModuleRevision, ResolveData }
import org.apache.ivy.plugins.latest.LatestStrategy import org.apache.ivy.plugins.latest.LatestStrategy
import org.apache.ivy.plugins.repository.file.{ FileRepository => IFileRepository, FileResource } import org.apache.ivy.plugins.repository.file.{ FileResource, FileRepository => IFileRepository }
import org.apache.ivy.plugins.repository.url.URLResource import org.apache.ivy.plugins.repository.url.URLResource
import org.apache.ivy.plugins.resolver._ import org.apache.ivy.plugins.resolver._
import org.apache.ivy.plugins.resolver.util.{ HasLatestStrategy, ResolvedResource } import org.apache.ivy.plugins.resolver.util.{ HasLatestStrategy, ResolvedResource }
@ -23,6 +22,8 @@ import org.apache.ivy.util.{ Message, StringUtils => IvyStringUtils }
import sbt.util.Logger import sbt.util.Logger
import sbt.librarymanagement._ import sbt.librarymanagement._
import scala.util.control.NonFatal
private[sbt] case class SbtChainResolver( private[sbt] case class SbtChainResolver(
name: String, name: String,
resolvers: Seq[DependencyResolver], resolvers: Seq[DependencyResolver],
@ -71,188 +72,240 @@ private[sbt] case class SbtChainResolver(
// //
// Ideally this could just skip the lookup, but unfortunately several artifacts in practice do not follow the // Ideally this could just skip the lookup, but unfortunately several artifacts in practice do not follow the
// correct behavior for packaging="pom" and so it is only skipped for source/javadoc classifiers. // correct behavior for packaging="pom" and so it is only skipped for source/javadoc classifiers.
override def locate(artifact: IArtifact) = override def locate(artifact: IArtifact): ArtifactOrigin =
if (IvySbt.hasImplicitClassifier(artifact)) null else super.locate(artifact) if (IvySbt.hasImplicitClassifier(artifact)) null else super.locate(artifact)
override def getDependency(dd: DependencyDescriptor, data: ResolveData) = { override def getDependency(dd: DependencyDescriptor, data: ResolveData): ResolvedModuleRevision = {
if (data.getOptions.getLog != LogOptions.LOG_QUIET) if (data.getOptions.getLog != LogOptions.LOG_QUIET)
Message.debug("Resolving " + dd.getDependencyRevisionId + " ...") Message.debug("Resolving " + dd.getDependencyRevisionId + " ...")
val gd = doGetDependency(dd, data) val gd = CustomSbtResolution.getDependency(dd, data)
val mod = IvySbt.resetArtifactResolver(gd) val mod = IvySbt.resetArtifactResolver(gd)
mod mod
} }
// Modified implementation of ChainResolver#getDependency.
// When the dependency is changing, it will check all resolvers on the chain /** Implements the custom sbt chain resolution with support for snapshots and caching. */
// regardless of what the "latest strategy" is set, and look for the published date private object CustomSbtResolution {
// or the module descriptor to sort them. def getCached(dd: DependencyDescriptor,
// This implementation also skips resolution if "return first" is set to true, data: ResolveData,
// and if a previously resolved or cached revision has been found. resolved0: Option[ResolvedModuleRevision]): Option[ResolvedModuleRevision] = {
def doGetDependency(dd: DependencyDescriptor, data0: ResolveData): ResolvedModuleRevision = { resolved0.orElse {
// useLatest - Means we should always download the JARs from the internet, no matter what. val resolverName = getName
// This will only be true *IF* the depenendency is dynamic/changing *and* latestSnapshots is true. Message.verbose(s"$resolverName: Checking cache for: $dd")
// If you find multiple candidates, Option(findModuleInCache(dd, data, true)).map { moduleRev =>
// - If `isReturnFirst` is true, you return the first value found Message.verbose(s"$resolverName: module revision found in cache: ${moduleRev.getId}")
// - If not, we will ATTEMPT to look at the publish date, which is not correctly discovered for Maven modules and forcedRevision(moduleRev)
// leads to undefined behavior.
val useLatest = (dd.isChanging || IvySbt.isChanging(dd.getDependencyRevisionId)) && updateOptions.latestSnapshots
if (useLatest) {
Message.verbose(s"$getName is changing. Checking all resolvers on the chain")
}
val data = new ResolveData(data0, doValidate(data0))
// Returns the value if we've already been resolved from some other branch of the resolution tree.
val resolved = Option(data.getCurrentResolvedModuleRevision)
// If we don't have any previously resolved date, we try to pull the value from the cache.
val resolvedOrCached =
resolved orElse {
Message.verbose(getName + ": Checking cache for: " + dd)
Option(findModuleInCache(dd, data, true)) map { mr =>
Message.verbose(getName + ": module revision found in cache: " + mr.getId)
forcedRevision(mr)
} }
} }
// Default value for resolution. We use this while we loop...
// If useLatest is true, we want to try to download from the internet so we DO NOT start with a valid value.
var temp: Option[ResolvedModuleRevision] =
if (useLatest) None
else resolvedOrCached
// Cast resolvers to something useful. TODO - we dropping anything here?
val resolvers = getResolvers.toArray.toVector collect { case x: DependencyResolver => x }
val interProjResolver = resolvers find { x =>
x.getName == ProjectResolver.InterProject
} }
// Here we do an attempt to resolve the artifact from each of the resolvers in the chain. /* Copy pasted from `IvyStringUtils` to handle `Throwable` */
// - If we have a return value already, AND isReturnFirst is true AND useLatest is false, we DO NOT resolve anything private def getStackTrace(e: Throwable): String = {
// - If we do not, try to resolve. if (e == null) return ""
// RETURNS: Left -> Error val baos = new ByteArrayOutputStream()
// Right -> Some(resolved module) // Found in this resolver, can use this result. val printWriter = new PrintWriter(baos)
// Right -> None // Do not use this resolver e.printStackTrace(printWriter)
lazy val results = resolvers map { x => printWriter.flush()
// if the revision is cached and isReturnFirst is set, don't bother hitting any resolvers, just return None for this guy. val stackTrace = new String(baos.toByteArray)
if (isReturnFirst && temp.isDefined && !useLatest) Right(None) printWriter.close()
else { stackTrace
// We actually do resolution. }
val resolver = x
val oldLatest: Option[LatestStrategy] = /** If None, module was not found. Otherwise, hit. */
setLatestIfRequired(resolver, Option(getLatestStrategy)) type TriedResolution = Option[(ResolvedModuleRevision, DependencyResolver)]
try {
val previouslyResolved = temp /** Attempts to resolve the artifact from each of the resolvers in the chain.
// if the module qualifies as changing, then resolve all resolvers *
if (useLatest) data.setCurrentResolvedModuleRevision(null) * Contract:
else data.setCurrentResolvedModuleRevision(temp.orNull) * 1. It doesn't resolve anything when there is a resolved module, `isReturnFirst` is
temp = Option(resolver.getDependency(dd, data)) * enabled and `useLatest` is false (meaning that resolution is pure, no SNAPSHOT).
Right( * 2. Otherwise, we try to resolve it.
if (temp eq previouslyResolved) None *
else if (useLatest) temp map { x => * @param resolved0 The perhaps already resolved module.
(reparseModuleDescriptor(dd, data, resolver, x), resolver) * @param useLatest Whether snapshot resolution should be enabled.
} else * @param data The resolve data to use.
temp map { x => * @param descriptor The dependency descriptor of the in-resolution module.
(forcedRevision(x), resolver) */
} def getResults(
) resolved0: Option[ResolvedModuleRevision],
} catch { useLatest: Boolean,
case ex: Exception => data: ResolveData,
Message.verbose( descriptor: DependencyDescriptor
"problem occurred while resolving " + dd + " with " + resolver ): Seq[Either[Throwable, TriedResolution]] = {
+ ": " + IvyStringUtils.getStackTrace(ex) var currentlyResolved = resolved0
)
Left(ex) def performResolution(
} finally { resolver: DependencyResolver): Option[(ResolvedModuleRevision, DependencyResolver)] = {
oldLatest map { _ => // Resolve all resolvers when the module is changing
doSetLatestStrategy(resolver, oldLatest) val previouslyResolved = currentlyResolved
if (useLatest) data.setCurrentResolvedModuleRevision(null)
else data.setCurrentResolvedModuleRevision(currentlyResolved.orNull)
currentlyResolved = Option(resolver.getDependency(descriptor, data))
if (currentlyResolved eq previouslyResolved) None
else if (useLatest) {
currentlyResolved.map(x =>
(reparseModuleDescriptor(descriptor, data, resolver, x), resolver))
} else currentlyResolved.map(x => (forcedRevision(x), resolver))
}
def reportError(throwable: Throwable, resolver: DependencyResolver): Unit = {
val trace = getStackTrace(throwable)
Message.verbose(s"problem occurred while resolving $descriptor with $resolver: $trace")
}
resolvers.map { (resolver: DependencyResolver) =>
// Return none when revision is cached and `isReturnFirst` is set
if (isReturnFirst && currentlyResolved.isDefined && !useLatest) Right(None)
else {
// We actually do resolution.
val oldLatest: Option[LatestStrategy] =
setLatestIfRequired(resolver, Option(getLatestStrategy))
try Right(performResolution(resolver))
catch { case NonFatal(t) => reportError(t, resolver); Left(t) } finally {
oldLatest.foreach(_ => doSetLatestStrategy(resolver, oldLatest))
checkInterrupted()
} }
checkInterrupted()
} }
} }
} }
lazy val errors = results collect { case Left(e) => e }
// If the value is arleady in cache, SORTED will be a Seq(None, None, ...) which means we'll fall over to the prevously cached or resolved version. private final val prefix = "Undefined resolution order"
val mrOpt: Option[ResolvedModuleRevision] = { def resolveLatest(foundRevisions: Seq[(ResolvedModuleRevision, DependencyResolver)],
val interProj: Option[ResolvedModuleRevision] = descriptor: DependencyDescriptor,
if (updateOptions.interProjectFirst) interProjResolver flatMap { x => data: ResolveData): Option[ResolvedModuleRevision] = {
Option(x.getDependency(dd, data))
} else None val sortedRevisions = foundRevisions.sortBy {
def foundRevisions: Vector[(ResolvedModuleRevision, DependencyResolver)] = results collect { case (rmr, resolver) =>
case Right(Some(x)) => x val publicationDate = rmr.getPublicationDate
} val descriptorDate = rmr.getDescriptor.getPublicationDate
def sorted = Message.warn(s"Sorting results from $rmr, using $publicationDate and $descriptorDate.")
if (useLatest)(foundRevisions // Just issue warning about issues with publication date, and fake one on it for now
.sortBy { val chosenPublicationDate = Option(publicationDate).orElse(Option(descriptorDate))
case (rmr, resolver) => chosenPublicationDate match {
Message.warn( case Some(date) => date.getTime
s"Sorting results from $rmr, using ${rmr.getPublicationDate} and ${rmr.getDescriptor.getPublicationDate}" case None =>
) val id = rmr.getId
// Just issue warning about issues with publication date, and fake one on it for now. val resolvedResource = (resolver.findIvyFileRef(descriptor, data), rmr.getDescriptor)
Option(rmr.getPublicationDate) orElse Option(rmr.getDescriptor.getPublicationDate) match { resolvedResource match {
case None => case (res: ResolvedResource, dmd: DefaultModuleDescriptor) =>
(resolver.findIvyFileRef(dd, data), rmr.getDescriptor) match { val resolvedPublicationDate = new java.util.Date(res.getLastModified)
case (null, _) => Message.debug(s"No publication date from resolver $resolver for $id.")
// In this instance, the dependency is specified by a direct URL or some other sort of "non-ivy" file Message.debug(s"Setting publication date to: $resolvedPublicationDate.")
if (dd.isChanging) dmd.setPublicationDate(resolvedPublicationDate)
Message.warn( res.getLastModified
s"Resolving a changing dependency (${rmr.getId}) with no ivy/pom file!, resolution order is undefined!" case (ivf, dmd) =>
) // The dependency is specified by a direct URL or some sort of non-ivy file
0L if (ivf == null && descriptor.isChanging)
case (ivf, dmd: DefaultModuleDescriptor) => Message.warn(s"$prefix: changing dependency $id with no ivy/pom file!")
val lmd = new java.util.Date(ivf.getLastModified) if (dmd == null)
Message.debug( Message.warn(s"$prefix: no publication date from resolver $resolver for $id")
s"Getting no publication date from resolver: ${resolver} for ${rmr.getId}, setting to: ${lmd}" 0L
)
dmd.setPublicationDate(lmd)
ivf.getLastModified
case _ =>
Message.warn(
s"Getting null publication date from resolver: ${resolver} for ${rmr.getId}, resolution order is undefined!"
)
0L
}
case Some(date) => // All other cases ok
date.getTime
} }
}) }
.reverse }
.headOption map {
case (rmr, resolver) =>
Message.warn(s"Choosing $resolver for ${rmr.getId}")
// Now that we know the real latest revision, let's force Ivy to use it
val artifactOpt = findFirstArtifactRef(rmr.getDescriptor, dd, data, resolver)
artifactOpt match {
case Some(artifactRef) =>
val systemMd = toSystem(rmr.getDescriptor)
getRepositoryCacheManager.cacheModuleDescriptor(
resolver,
artifactRef,
toSystem(dd),
systemMd.getAllArtifacts.head,
None.orNull,
getCacheOptions(data)
)
case None => // do nothing. There are modules without artifacts
}
rmr
} else
foundRevisions.reverse.headOption map { _._1 } // we have to reverse because resolvers are hit in reverse order.
interProj orElse sorted orElse resolvedOrCached val firstHit = sortedRevisions.reverse.headOption
} firstHit.map { hit =>
mrOpt match { val (resolvedModule, resolver) = hit
case None if errors.size == 1 => Message.warn(s"Choosing $resolver for ${resolvedModule.getId}")
errors.head match { // Now that we know the real latest revision, let's force Ivy to use it
case e: RuntimeException => throw e val resolvedDescriptor = resolvedModule.getDescriptor
case e: ParseException => throw e val artifactOpt = findFirstArtifactRef(resolvedDescriptor, descriptor, data, resolver)
case e: Throwable => throw new RuntimeException(e.toString, e) // If `None` do nothing -- modules without artifacts. Otherwise cache.
artifactOpt.foreach { artifactRef =>
val dep = toSystem(descriptor)
val first = toSystem(resolvedDescriptor).getAllArtifacts.head
val options = getCacheOptions(data)
val cacheManager = getRepositoryCacheManager
cacheManager.cacheModuleDescriptor(resolver, artifactRef, dep, first, null, options)
} }
case None if errors.size > 1 => resolvedModule
val err = }
(errors.toList map { IvyStringUtils.getErrorMessage }).mkString("\n\t", "\n\t", "\n") }
throw new RuntimeException(s"several problems occurred while resolving $dd:$err")
case _ => def resolveByAllMeans(
if (resolved == mrOpt) resolved.orNull cachedModule: Option[ResolvedModuleRevision],
else (mrOpt map { resolvedRevision }).orNull useLatest: Boolean,
interResolver: Option[DependencyResolver],
resolveModules: () => Seq[Either[Throwable, TriedResolution]],
dd: DependencyDescriptor,
data: ResolveData
): Option[ResolvedModuleRevision] = {
val internallyResolved: Option[ResolvedModuleRevision] = {
if (!updateOptions.interProjectFirst) None
else interResolver.flatMap(resolver => Option(resolver.getDependency(dd, data)))
}
val internalOrExternal = internallyResolved.orElse {
val foundRevisions: Seq[(ResolvedModuleRevision, DependencyResolver)] =
resolveModules().collect { case Right(Some(x)) => x }
if (useLatest) resolveLatest(foundRevisions, dd, data)
else foundRevisions.reverse.headOption.map(_._1) // Resolvers are hit in reverse order
}
internalOrExternal.orElse(cachedModule)
}
// The ivy implementation guarantees that all resolvers implement `DependencyResolver`
def getDependencyResolvers: Vector[DependencyResolver] =
getResolvers.toArray.collect { case r: DependencyResolver => r }.toVector
def findInterProjectResolver(resolvers: Seq[DependencyResolver]): Option[DependencyResolver] =
resolvers.find(_.getName == ProjectResolver.InterProject)
/** Gets the dependency for a given descriptor with the pertinent resolve data.
*
* This is a custom sbt chain operation that produces better error output and deals with
* cases that the conventional ivy resolver does not. It accumulates the resolution of
* several resolvers and returns the module which fits the provided resolution strategy.
*
* These are the differences with regard to the default ivy [[ChainResolver]]:
* 1. It skips resolution if "return first" is set to true.
* 2. It skips resolution if a previously resolved or cached resolution is found.
* 3. It always checks all the resolvers and compares timestamps for changing dependencies
* if and only if `latestSnapshots` is enabled in the update options, regardless of what
* the latest strategies are (http://ant.apache.org/ivy/history/2.3.0/settings/latest-strategies.html).
* See https://github.com/sbt/sbt/pull/1520 for more information on this topic.
*
* Note the tradeoff here in SNAPSHOTs: correctness vs slowness.
*/
def getDependency(dd: DependencyDescriptor, data0: ResolveData): ResolvedModuleRevision = {
val isDynamic = dd.isChanging || IvySbt.isChanging(dd.getDependencyRevisionId)
val useLatest = isDynamic && updateOptions.latestSnapshots
if (useLatest) Message.verbose(s"$getName is changing. Checking all resolvers on the chain.")
/* Get the resolved module descriptor from:
* 1. An already resolved branch of the resolution tree.
* 2. The value from the cache. */
val data = new ResolveData(data0, doValidate(data0))
val resolved0 = Option(data.getCurrentResolvedModuleRevision)
val resolvedOrCached = getCached(dd, data0, resolved0)
val cached: Option[ResolvedModuleRevision] = if (useLatest) None else resolvedOrCached
val resolvers = getDependencyResolvers
val interResolver = findInterProjectResolver(resolvers)
// TODO: Please, change `Option` return types so that this goes away
lazy val results = getResults(cached, useLatest, data, dd)
lazy val errors = results.collect { case Left(t) => t }
val runResolution = () => results
val resolved = resolveByAllMeans(cached, useLatest, interResolver, runResolution, dd, data)
resolved match {
case None if errors.size == 1 =>
errors.head match {
case e: RuntimeException => throw e
case e: ParseException => throw e
case e: Throwable => throw new RuntimeException(e.toString, e)
}
case None if errors.size > 1 =>
val traces = errors.toList.map(e => IvyStringUtils.getErrorMessage(e))
val msg = s"Resolution failed several times for $dd:"
throw new RuntimeException(s"$msg: ${traces.mkString("\n\t", "\n\t", "\n")}")
case _ =>
// Can be either `None` with empty error or `Some`
if (resolved0 == resolved) resolved0.orNull
else resolved.map(resolvedRevision).orNull
}
} }
} }
// Ivy seem to not want to use the module descriptor found at the latest resolver // Ivy seem to not want to use the module descriptor found at the latest resolver
private[this] def reparseModuleDescriptor( private[this] def reparseModuleDescriptor(
dd: DependencyDescriptor, dd: DependencyDescriptor,