luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1653 from sbt/wip/fix-1649 

Fixes #1649. Exclusion rules and other cached resolution fixes
This commit is contained in:
Josh Suereth 2014-10-12 12:01:22 -04:00
parent 855dff21be 68b3f8d5ed
commit adb4dc64e7
7 changed files with 122 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ivy/src/main/scala/sbt/Ivy.scala
View File

@ -4,7 +4,7 @@
package sbt
import Resolver.PluginPattern
import ivyint.{ CachedResolutionResolveEngine, CachedResolutionResolveCache }
import ivyint.{ CachedResolutionResolveEngine, CachedResolutionResolveCache, SbtDefaultDependencyDescriptor }
import java.io.File
import java.net.URI
@ -570,7 +570,9 @@ private[sbt] object IvySbt {
/** Transforms an sbt ModuleID into an Ivy DefaultDependencyDescriptor.*/
def convertDependency(moduleID: DefaultModuleDescriptor, dependency: ModuleID, parser: CustomXmlParser.CustomParser): DefaultDependencyDescriptor =
{
val dependencyDescriptor = new DefaultDependencyDescriptor(moduleID, toID(dependency), dependency.isForce, dependency.isChanging, dependency.isTransitive)
val dependencyDescriptor = new DefaultDependencyDescriptor(moduleID, toID(dependency), dependency.isForce, dependency.isChanging, dependency.isTransitive) with SbtDefaultDependencyDescriptor {
def dependencyModuleId = dependency
}
dependency.configurations match {
case None => // The configuration for this dependency was not explicitly specified, so use the default
parser.parseDepsConfs(parser.getDefaultConf, dependencyDescriptor)

28
ivy/src/main/scala/sbt/IvyActions.scala
View File

@ -155,13 +155,13 @@ object IvyActions {
private[sbt] def updateEither(module: IvySbt#Module, configuration: UpdateConfiguration,
uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): Either[UnresolvedWarning, UpdateReport] =
module.withModule(log) {
case (ivy, md, default) if module.owner.configuration.updateOptions.cachedResolution =>
case (ivy, md, default) if module.owner.configuration.updateOptions.cachedResolution && depDir.isDefined =>
ivy.getResolveEngine match {
case x: CachedResolutionResolveEngine =>
val resolveOptions = new ResolveOptions
val resolveId = ResolveOptions.getDefaultResolveId(md)
resolveOptions.setResolveId(resolveId)
x.customResolve(md, logicalClock, resolveOptions, depDir getOrElse { sys.error("dependency base directory is not specified") }, log) match {
x.customResolve(md, configuration.missingOk, logicalClock, resolveOptions, depDir getOrElse { sys.error("dependency base directory is not specified") }, log) match {
case Left(x) =>
Left(UnresolvedWarning(x, uwconfig))
case Right(uReport) =>
@ -198,17 +198,31 @@ object IvyActions {
def grouped[T](grouping: ModuleID => T)(mods: Seq[ModuleID]): Map[T, Set[String]] =
mods groupBy (grouping) mapValues (_.map(_.revision).toSet)
@deprecated("This is no longer public.", "0.13.6")
def transitiveScratch(ivySbt: IvySbt, label: String, config: GetClassifiersConfiguration, log: Logger): UpdateReport =
transitiveScratch(ivySbt, label, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log)
private[sbt] def transitiveScratch(ivySbt: IvySbt, label: String, config: GetClassifiersConfiguration,
uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): UpdateReport =
{
import config.{ configuration => c, ivyScala, module => mod }
import mod.{ id, modules => deps }
val base = restrictedCopy(id, true).copy(name = id.name + "$" + label)
val module = new ivySbt.Module(InlineConfiguration(base, ModuleInfo(base.name), deps).copy(ivyScala = ivyScala))
val report = update(module, c, log)
val report = updateEither(module, c, uwconfig, logicalClock, depDir, log) match {
case Right(r) => r
case Left(w) =>
throw w.resolveException
}
val newConfig = config.copy(module = mod.copy(modules = report.allModules))
updateClassifiers(ivySbt, newConfig, log)
updateClassifiers(ivySbt, newConfig, uwconfig, logicalClock, depDir, log)
}
@deprecated("This is no longer public.", "0.13.6")
def updateClassifiers(ivySbt: IvySbt, config: GetClassifiersConfiguration, log: Logger): UpdateReport =
updateClassifiers(ivySbt, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log)
private[sbt] def updateClassifiers(ivySbt: IvySbt, config: GetClassifiersConfiguration,
uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): UpdateReport =
{
import config.{ configuration => c, module => mod, _ }
import mod.{ configurations => confs, _ }
@ -218,7 +232,11 @@ object IvyActions {
val base = restrictedCopy(id, true).copy(name = id.name + classifiers.mkString("$", "_", ""))
val module = new ivySbt.Module(InlineConfiguration(base, ModuleInfo(base.name), deps).copy(ivyScala = ivyScala, configurations = confs))
val upConf = new UpdateConfiguration(c.retrieve, true, c.logging)
update(module, upConf, log)
updateEither(module, upConf, uwconfig, logicalClock, depDir, log) match {
case Right(r) => r
case Left(w) =>
throw w.resolveException
}
}
def classifiedArtifacts(classifiers: Seq[String], exclude: Map[ModuleID, Set[String]])(m: ModuleID): Option[ModuleID] =
{

12
ivy/src/main/scala/sbt/IvyRetrieve.scala
View File

@ -13,6 +13,7 @@ import module.id.{ ModuleRevisionId, ModuleId => IvyModuleId }
import report.{ ArtifactDownloadReport, ConfigurationResolveReport, ResolveReport }
import resolve.{ IvyNode, IvyNodeCallers }
import IvyNodeCallers.{ Caller => IvyCaller }
import ivyint.SbtDefaultDependencyDescriptor
object IvyRetrieve {
def reports(report: ResolveReport): Seq[ConfigurationResolveReport] =
@ -77,11 +78,14 @@ object IvyRetrieve {
case x if nonEmptyString(x).isDefined => x
}
val ddOpt = Option(caller.getDependencyDescriptor)
val (extraAttributes, isForce, isChanging, isTransitive) = ddOpt match {
case Some(dd) => (toExtraAttributes(dd.getExtraAttributes), dd.isForce, dd.isChanging, dd.isTransitive)
case None => (Map.empty[String, String], false, false, true)
val (extraAttributes, isForce, isChanging, isTransitive, isDirectlyForce) = ddOpt match {
case Some(dd: SbtDefaultDependencyDescriptor) =>
val mod = dd.dependencyModuleId
(toExtraAttributes(dd.getExtraAttributes), mod.isForce, mod.isChanging, mod.isTransitive, mod.isForce)
case Some(dd) => (toExtraAttributes(dd.getExtraAttributes), dd.isForce, dd.isChanging, dd.isTransitive, false)
case None => (Map.empty[String, String], false, false, true, false)
}
new Caller(m, callerConfigurations, extraAttributes, isForce, isChanging, isTransitive)
new Caller(m, callerConfigurations, extraAttributes, isForce, isChanging, isTransitive, isDirectlyForce)
}
val revId = dep.getResolvedId
val moduleId = toModuleID(revId)

5
ivy/src/main/scala/sbt/LogicalClock.scala
View File

@ -9,6 +9,11 @@ trait LogicalClock {
}
object LogicalClock {
def apply(hashCode: Int): LogicalClock = {
def intToByteArray(x: Int): Array[Byte] =
Array((x >>> 24).toByte, (x >> 16 & 0xff).toByte, (x >> 8 & 0xff).toByte, (x & 0xff).toByte)
apply(Hash.toHex(intToByteArray(hashCode)))
}
def apply(x: String): LogicalClock = new LogicalClock {
override def toString: String = x
}

14
ivy/src/main/scala/sbt/UpdateReport.scala
View File

@ -34,6 +34,12 @@ final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[Con
/** Gets the names of all resolved configurations. This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */
def allConfigurations: Seq[String] = configurations.map(_.configuration)
private[sbt] def withStats(us: UpdateStats): UpdateReport =
new UpdateReport(this.cachedDescriptor,
this.configurations,
us,
this.stamps)
}
/**
@ -190,7 +196,8 @@ final class Caller(
val callerExtraAttributes: Map[String, String],
val isForceDependency: Boolean,
val isChangingDependency: Boolean,
val isTransitiveDependency: Boolean) {
val isTransitiveDependency: Boolean,
val isDirectlyForceDependency: Boolean) {
override def toString: String =
s"$caller"
}
@ -267,4 +274,9 @@ object UpdateReport {
}
final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downloadSize: Long, val cached: Boolean) {
override def toString = Seq("Resolve time: " + resolveTime + " ms", "Download time: " + downloadTime + " ms", "Download size: " + downloadSize + " bytes").mkString(", ")
private[sbt] def withCached(c: Boolean): UpdateStats =
new UpdateStats(resolveTime = this.resolveTime,
downloadTime = this.downloadTime,
downloadSize = this.downloadSize,
cached = c)
}

79
ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala
View File

@ -11,7 +11,7 @@ import org.apache.ivy.core
import core.resolve._
import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId }
import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport }
import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration }
import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule }
import core.{ IvyPatternHelper, LogOptions }
import org.apache.ivy.util.Message
import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo }
@ -36,6 +36,9 @@ private[sbt] class CachedResolutionResolveCache() {
val mds =
if (mrid0.getOrganisation == sbtOrgTemp) Vector(md0)
else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 }
updateReportCache.remove(md0.getModuleRevisionId)
directDependencyCache.remove(md0.getModuleRevisionId)
mds foreach { md =>
updateReportCache.remove(md.getModuleRevisionId)
directDependencyCache.remove(md.getModuleRevisionId)
@ -59,11 +62,28 @@ private[sbt] class CachedResolutionResolveCache() {
}
def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) =
{
def excludeRuleString(rule: ExcludeRule): String =
s"""Exclude(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})"""
def includeRuleString(rule: IncludeRule): String =
s"""Include(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})"""
val mrid = dd.getDependencyRevisionId
val confMap = (dd.getModuleConfigurations map { conf =>
conf + "->(" + dd.getDependencyConfigurations(conf).mkString(",") + ")"
})
val depsString = mrid.toString + ";" + confMap.mkString(";")
val exclusions = (dd.getModuleConfigurations.toVector flatMap { conf =>
dd.getExcludeRules(conf).toVector match {
case Vector() => None
case rules => Some(conf + "->(" + (rules map excludeRuleString).mkString(",") + ")")
}
})
val inclusions = (dd.getModuleConfigurations.toVector flatMap { conf =>
dd.getIncludeRules(conf).toVector match {
case Vector() => None
case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")")
}
})
val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" +
s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};"""
val sha1 = Hash.toHex(Hash(depsString))
val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor {
def targetModuleRevisionId: ModuleRevisionId = mrid
@ -92,14 +112,26 @@ private[sbt] class CachedResolutionResolveCache() {
def loadMiniGraphFromFile: Option[Either[ResolveException, UpdateReport]] =
(if (staticGraphPath.exists) Some(staticGraphPath)
else if (dynamicGraphPath.exists) Some(dynamicGraphPath)
else None) map { path =>
log.debug(s"parsing ${path.getAbsolutePath.toString}")
val ur = JsonUtil.parseUpdateReport(md, path, cachedDescriptor, log)
updateReportCache(md.getModuleRevisionId) = Right(ur)
Right(ur)
else None) match {
case Some(path) =>
log.debug(s"parsing ${path.getAbsolutePath.toString}")
val ur = JsonUtil.parseUpdateReport(md, path, cachedDescriptor, log)
if (ur.allFiles forall { _.exists }) {
updateReportCache(md.getModuleRevisionId) = Right(ur)
Some(Right(ur))
} else {
log.debug(s"some files are missing from the cache, so invalidating the minigraph")
IO.delete(path)
None
}
case _ => None
}
(updateReportCache.get(mrid) orElse loadMiniGraphFromFile) match {
case Some(result) => result
case Some(result) =>
result match {
case Right(ur) => Right(ur.withStats(ur.stats.withCached(true)))
case x => x
}
case None =>
f match {
case Right(ur) =>
@ -163,19 +195,24 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
private[sbt] def projectResolver: Option[ProjectResolver]
private[sbt] def makeInstance: Ivy
// Return sbt's UpdateReport.
def customResolve(md0: ModuleDescriptor, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = {
/**
* This returns sbt's UpdateReport structure.
* missingOk allows sbt to call this with classifiers that may or may not exist, and grab the JARs.
*/
def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = {
import Path._
val start = System.currentTimeMillis
val miniGraphPath = depDir / "module"
val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId)
val cache = cachedResolutionResolveCache
cache.directDependencyCache.remove(md0.getModuleRevisionId)
val mds = cache.buildArtificialModuleDescriptors(md0, projectResolver)
def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] =
{
val options1 = new ResolveOptions(options0)
val i = makeInstance
var rr = i.resolve(md, options1)
if (!rr.hasError) Right(IvyRetrieve.updateReport(rr, cachedDescriptor))
if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor))
else {
val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct
val failedPaths = ListMap(rr.getUnresolvedDependencies map { node =>
@ -196,7 +233,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
doWork(md)
}
}
val uReport = mergeResults(md0, results, log)
val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, log)
val cacheManager = getSettings.getResolutionCacheManager
cacheManager.saveResolvedModuleDescriptor(md0)
val prop0 = ""
@ -204,9 +241,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
IO.write(ivyPropertiesInCache0, prop0)
uReport
}
def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], log: Logger): Either[ResolveException, UpdateReport] =
if (results exists { _.isLeft }) Left(mergeErrors(md0, results collect { case Left(re) => re }, log))
else Right(mergeReports(md0, results collect { case Right(ur) => ur }, log))
def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], missingOk: Boolean, resolveTime: Long, log: Logger): Either[ResolveException, UpdateReport] =
if (!missingOk && (results exists { _.isLeft })) Left(mergeErrors(md0, results collect { case Left(re) => re }, log))
else Right(mergeReports(md0, results collect { case Right(ur) => ur }, resolveTime, log))
def mergeErrors(md0: ModuleDescriptor, errors: Vector[ResolveException], log: Logger): ResolveException =
{
val messages = errors flatMap { _.messages }
@ -220,11 +257,12 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
}
new ResolveException(messages, failed, ListMap(failedPaths: _*))
}
def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], log: Logger): UpdateReport =
def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], resolveTime: Long, log: Logger): UpdateReport =
{
val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId)
val rootModuleConfigs = md0.getConfigurations.toVector
val stats = new UpdateStats(0L, 0L, 0L, false)
val cachedReports = reports filter { !_.stats.cached }
val stats = new UpdateStats(resolveTime, (cachedReports map { _.stats.downloadTime }).sum, (cachedReports map { _.stats.downloadSize }).sum, false)
val configReports = rootModuleConfigs map { conf =>
val crs = reports flatMap { _.configurations filter { _.configuration == conf.getName } }
mergeConfigurationReports(conf.getName, crs, log)
@ -275,10 +313,13 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
val name = head.module.name
log.debug(s"- conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")"))
def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) =
conflicts find { m =>
(conflicts find { m =>
m.callers.exists { _.isDirectlyForceDependency }
} orElse (conflicts find { m =>
m.callers.exists { _.isForceDependency }
} match {
})) match {
case Some(m) =>
log.debug(s"- forced dependency: $m")
(Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString)
case None =>
val strategy = lcm.getStrategy

9
ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala Normal file
View File

@ -0,0 +1,9 @@
package sbt
package ivyint
import org.apache.ivy.core
import core.module.descriptor.DefaultDependencyDescriptor
trait SbtDefaultDependencyDescriptor { self: DefaultDependencyDescriptor =>
def dependencyModuleId: ModuleID
}