diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 4bf6ffad6..c0e7d508a 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -4,7 +4,7 @@ package sbt import Resolver.PluginPattern -import ivyint.{ CachedResolutionResolveEngine, CachedResolutionResolveCache } +import ivyint.{ CachedResolutionResolveEngine, CachedResolutionResolveCache, SbtDefaultDependencyDescriptor } import java.io.File import java.net.URI @@ -570,7 +570,9 @@ private[sbt] object IvySbt { /** Transforms an sbt ModuleID into an Ivy DefaultDependencyDescriptor.*/ def convertDependency(moduleID: DefaultModuleDescriptor, dependency: ModuleID, parser: CustomXmlParser.CustomParser): DefaultDependencyDescriptor = { - val dependencyDescriptor = new DefaultDependencyDescriptor(moduleID, toID(dependency), dependency.isForce, dependency.isChanging, dependency.isTransitive) + val dependencyDescriptor = new DefaultDependencyDescriptor(moduleID, toID(dependency), dependency.isForce, dependency.isChanging, dependency.isTransitive) with SbtDefaultDependencyDescriptor { + def dependencyModuleId = dependency + } dependency.configurations match { case None => // The configuration for this dependency was not explicitly specified, so use the default parser.parseDepsConfs(parser.getDefaultConf, dependencyDescriptor) diff --git a/ivy/src/main/scala/sbt/IvyActions.scala b/ivy/src/main/scala/sbt/IvyActions.scala index f5e57336d..7dd07df7d 100644 --- a/ivy/src/main/scala/sbt/IvyActions.scala +++ b/ivy/src/main/scala/sbt/IvyActions.scala @@ -155,13 +155,13 @@ object IvyActions { private[sbt] def updateEither(module: IvySbt#Module, configuration: UpdateConfiguration, uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): Either[UnresolvedWarning, UpdateReport] = module.withModule(log) { - case (ivy, md, default) if module.owner.configuration.updateOptions.cachedResolution => + case (ivy, md, default) if module.owner.configuration.updateOptions.cachedResolution && depDir.isDefined => ivy.getResolveEngine match { case x: CachedResolutionResolveEngine => val resolveOptions = new ResolveOptions val resolveId = ResolveOptions.getDefaultResolveId(md) resolveOptions.setResolveId(resolveId) - x.customResolve(md, logicalClock, resolveOptions, depDir getOrElse { sys.error("dependency base directory is not specified") }, log) match { + x.customResolve(md, configuration.missingOk, logicalClock, resolveOptions, depDir getOrElse { sys.error("dependency base directory is not specified") }, log) match { case Left(x) => Left(UnresolvedWarning(x, uwconfig)) case Right(uReport) => @@ -198,17 +198,31 @@ object IvyActions { def grouped[T](grouping: ModuleID => T)(mods: Seq[ModuleID]): Map[T, Set[String]] = mods groupBy (grouping) mapValues (_.map(_.revision).toSet) + @deprecated("This is no longer public.", "0.13.6") def transitiveScratch(ivySbt: IvySbt, label: String, config: GetClassifiersConfiguration, log: Logger): UpdateReport = + transitiveScratch(ivySbt, label, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) + + private[sbt] def transitiveScratch(ivySbt: IvySbt, label: String, config: GetClassifiersConfiguration, + uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): UpdateReport = { import config.{ configuration => c, ivyScala, module => mod } import mod.{ id, modules => deps } val base = restrictedCopy(id, true).copy(name = id.name + "$" + label) val module = new ivySbt.Module(InlineConfiguration(base, ModuleInfo(base.name), deps).copy(ivyScala = ivyScala)) - val report = update(module, c, log) + val report = updateEither(module, c, uwconfig, logicalClock, depDir, log) match { + case Right(r) => r + case Left(w) => + throw w.resolveException + } val newConfig = config.copy(module = mod.copy(modules = report.allModules)) - updateClassifiers(ivySbt, newConfig, log) + updateClassifiers(ivySbt, newConfig, uwconfig, logicalClock, depDir, log) } + @deprecated("This is no longer public.", "0.13.6") def updateClassifiers(ivySbt: IvySbt, config: GetClassifiersConfiguration, log: Logger): UpdateReport = + updateClassifiers(ivySbt, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) + + private[sbt] def updateClassifiers(ivySbt: IvySbt, config: GetClassifiersConfiguration, + uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): UpdateReport = { import config.{ configuration => c, module => mod, _ } import mod.{ configurations => confs, _ } @@ -218,7 +232,11 @@ object IvyActions { val base = restrictedCopy(id, true).copy(name = id.name + classifiers.mkString("$", "_", "")) val module = new ivySbt.Module(InlineConfiguration(base, ModuleInfo(base.name), deps).copy(ivyScala = ivyScala, configurations = confs)) val upConf = new UpdateConfiguration(c.retrieve, true, c.logging) - update(module, upConf, log) + updateEither(module, upConf, uwconfig, logicalClock, depDir, log) match { + case Right(r) => r + case Left(w) => + throw w.resolveException + } } def classifiedArtifacts(classifiers: Seq[String], exclude: Map[ModuleID, Set[String]])(m: ModuleID): Option[ModuleID] = { diff --git a/ivy/src/main/scala/sbt/IvyRetrieve.scala b/ivy/src/main/scala/sbt/IvyRetrieve.scala index 1e74f3955..f453efb1f 100644 --- a/ivy/src/main/scala/sbt/IvyRetrieve.scala +++ b/ivy/src/main/scala/sbt/IvyRetrieve.scala @@ -13,6 +13,7 @@ import module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import report.{ ArtifactDownloadReport, ConfigurationResolveReport, ResolveReport } import resolve.{ IvyNode, IvyNodeCallers } import IvyNodeCallers.{ Caller => IvyCaller } +import ivyint.SbtDefaultDependencyDescriptor object IvyRetrieve { def reports(report: ResolveReport): Seq[ConfigurationResolveReport] = @@ -77,11 +78,14 @@ object IvyRetrieve { case x if nonEmptyString(x).isDefined => x } val ddOpt = Option(caller.getDependencyDescriptor) - val (extraAttributes, isForce, isChanging, isTransitive) = ddOpt match { - case Some(dd) => (toExtraAttributes(dd.getExtraAttributes), dd.isForce, dd.isChanging, dd.isTransitive) - case None => (Map.empty[String, String], false, false, true) + val (extraAttributes, isForce, isChanging, isTransitive, isDirectlyForce) = ddOpt match { + case Some(dd: SbtDefaultDependencyDescriptor) => + val mod = dd.dependencyModuleId + (toExtraAttributes(dd.getExtraAttributes), mod.isForce, mod.isChanging, mod.isTransitive, mod.isForce) + case Some(dd) => (toExtraAttributes(dd.getExtraAttributes), dd.isForce, dd.isChanging, dd.isTransitive, false) + case None => (Map.empty[String, String], false, false, true, false) } - new Caller(m, callerConfigurations, extraAttributes, isForce, isChanging, isTransitive) + new Caller(m, callerConfigurations, extraAttributes, isForce, isChanging, isTransitive, isDirectlyForce) } val revId = dep.getResolvedId val moduleId = toModuleID(revId) diff --git a/ivy/src/main/scala/sbt/LogicalClock.scala b/ivy/src/main/scala/sbt/LogicalClock.scala index 1056f6560..d4e1d4cc3 100644 --- a/ivy/src/main/scala/sbt/LogicalClock.scala +++ b/ivy/src/main/scala/sbt/LogicalClock.scala @@ -9,6 +9,11 @@ trait LogicalClock { } object LogicalClock { + def apply(hashCode: Int): LogicalClock = { + def intToByteArray(x: Int): Array[Byte] = + Array((x >>> 24).toByte, (x >> 16 & 0xff).toByte, (x >> 8 & 0xff).toByte, (x & 0xff).toByte) + apply(Hash.toHex(intToByteArray(hashCode))) + } def apply(x: String): LogicalClock = new LogicalClock { override def toString: String = x } diff --git a/ivy/src/main/scala/sbt/UpdateReport.scala b/ivy/src/main/scala/sbt/UpdateReport.scala index 443c84ed3..03ee2fe31 100644 --- a/ivy/src/main/scala/sbt/UpdateReport.scala +++ b/ivy/src/main/scala/sbt/UpdateReport.scala @@ -34,6 +34,12 @@ final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[Con /** Gets the names of all resolved configurations. This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */ def allConfigurations: Seq[String] = configurations.map(_.configuration) + + private[sbt] def withStats(us: UpdateStats): UpdateReport = + new UpdateReport(this.cachedDescriptor, + this.configurations, + us, + this.stamps) } /** @@ -190,7 +196,8 @@ final class Caller( val callerExtraAttributes: Map[String, String], val isForceDependency: Boolean, val isChangingDependency: Boolean, - val isTransitiveDependency: Boolean) { + val isTransitiveDependency: Boolean, + val isDirectlyForceDependency: Boolean) { override def toString: String = s"$caller" } @@ -267,4 +274,9 @@ object UpdateReport { } final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downloadSize: Long, val cached: Boolean) { override def toString = Seq("Resolve time: " + resolveTime + " ms", "Download time: " + downloadTime + " ms", "Download size: " + downloadSize + " bytes").mkString(", ") + private[sbt] def withCached(c: Boolean): UpdateStats = + new UpdateStats(resolveTime = this.resolveTime, + downloadTime = this.downloadTime, + downloadSize = this.downloadSize, + cached = c) } \ No newline at end of file diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 4b6cd3e9e..09da3b501 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -11,7 +11,7 @@ import org.apache.ivy.core import core.resolve._ import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport } -import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration } +import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule } import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.Message import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } @@ -36,6 +36,9 @@ private[sbt] class CachedResolutionResolveCache() { val mds = if (mrid0.getOrganisation == sbtOrgTemp) Vector(md0) else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 } + + updateReportCache.remove(md0.getModuleRevisionId) + directDependencyCache.remove(md0.getModuleRevisionId) mds foreach { md => updateReportCache.remove(md.getModuleRevisionId) directDependencyCache.remove(md.getModuleRevisionId) @@ -59,11 +62,28 @@ private[sbt] class CachedResolutionResolveCache() { } def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) = { + def excludeRuleString(rule: ExcludeRule): String = + s"""Exclude(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" + def includeRuleString(rule: IncludeRule): String = + s"""Include(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" val mrid = dd.getDependencyRevisionId val confMap = (dd.getModuleConfigurations map { conf => conf + "->(" + dd.getDependencyConfigurations(conf).mkString(",") + ")" }) - val depsString = mrid.toString + ";" + confMap.mkString(";") + val exclusions = (dd.getModuleConfigurations.toVector flatMap { conf => + dd.getExcludeRules(conf).toVector match { + case Vector() => None + case rules => Some(conf + "->(" + (rules map excludeRuleString).mkString(",") + ")") + } + }) + val inclusions = (dd.getModuleConfigurations.toVector flatMap { conf => + dd.getIncludeRules(conf).toVector match { + case Vector() => None + case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")") + } + }) + val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" + + s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};""" val sha1 = Hash.toHex(Hash(depsString)) val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor { def targetModuleRevisionId: ModuleRevisionId = mrid @@ -92,14 +112,26 @@ private[sbt] class CachedResolutionResolveCache() { def loadMiniGraphFromFile: Option[Either[ResolveException, UpdateReport]] = (if (staticGraphPath.exists) Some(staticGraphPath) else if (dynamicGraphPath.exists) Some(dynamicGraphPath) - else None) map { path => - log.debug(s"parsing ${path.getAbsolutePath.toString}") - val ur = JsonUtil.parseUpdateReport(md, path, cachedDescriptor, log) - updateReportCache(md.getModuleRevisionId) = Right(ur) - Right(ur) + else None) match { + case Some(path) => + log.debug(s"parsing ${path.getAbsolutePath.toString}") + val ur = JsonUtil.parseUpdateReport(md, path, cachedDescriptor, log) + if (ur.allFiles forall { _.exists }) { + updateReportCache(md.getModuleRevisionId) = Right(ur) + Some(Right(ur)) + } else { + log.debug(s"some files are missing from the cache, so invalidating the minigraph") + IO.delete(path) + None + } + case _ => None } (updateReportCache.get(mrid) orElse loadMiniGraphFromFile) match { - case Some(result) => result + case Some(result) => + result match { + case Right(ur) => Right(ur.withStats(ur.stats.withCached(true))) + case x => x + } case None => f match { case Right(ur) => @@ -163,19 +195,24 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { private[sbt] def projectResolver: Option[ProjectResolver] private[sbt] def makeInstance: Ivy - // Return sbt's UpdateReport. - def customResolve(md0: ModuleDescriptor, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { + /** + * This returns sbt's UpdateReport structure. + * missingOk allows sbt to call this with classifiers that may or may not exist, and grab the JARs. + */ + def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { import Path._ + val start = System.currentTimeMillis val miniGraphPath = depDir / "module" val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) val cache = cachedResolutionResolveCache + cache.directDependencyCache.remove(md0.getModuleRevisionId) val mds = cache.buildArtificialModuleDescriptors(md0, projectResolver) def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = { val options1 = new ResolveOptions(options0) val i = makeInstance var rr = i.resolve(md, options1) - if (!rr.hasError) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) + if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) else { val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct val failedPaths = ListMap(rr.getUnresolvedDependencies map { node => @@ -196,7 +233,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { doWork(md) } } - val uReport = mergeResults(md0, results, log) + val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, log) val cacheManager = getSettings.getResolutionCacheManager cacheManager.saveResolvedModuleDescriptor(md0) val prop0 = "" @@ -204,9 +241,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { IO.write(ivyPropertiesInCache0, prop0) uReport } - def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], log: Logger): Either[ResolveException, UpdateReport] = - if (results exists { _.isLeft }) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) - else Right(mergeReports(md0, results collect { case Right(ur) => ur }, log)) + def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], missingOk: Boolean, resolveTime: Long, log: Logger): Either[ResolveException, UpdateReport] = + if (!missingOk && (results exists { _.isLeft })) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) + else Right(mergeReports(md0, results collect { case Right(ur) => ur }, resolveTime, log)) def mergeErrors(md0: ModuleDescriptor, errors: Vector[ResolveException], log: Logger): ResolveException = { val messages = errors flatMap { _.messages } @@ -220,11 +257,12 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } new ResolveException(messages, failed, ListMap(failedPaths: _*)) } - def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], log: Logger): UpdateReport = + def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], resolveTime: Long, log: Logger): UpdateReport = { val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) val rootModuleConfigs = md0.getConfigurations.toVector - val stats = new UpdateStats(0L, 0L, 0L, false) + val cachedReports = reports filter { !_.stats.cached } + val stats = new UpdateStats(resolveTime, (cachedReports map { _.stats.downloadTime }).sum, (cachedReports map { _.stats.downloadSize }).sum, false) val configReports = rootModuleConfigs map { conf => val crs = reports flatMap { _.configurations filter { _.configuration == conf.getName } } mergeConfigurationReports(conf.getName, crs, log) @@ -275,10 +313,13 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val name = head.module.name log.debug(s"- conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")")) def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) = - conflicts find { m => + (conflicts find { m => + m.callers.exists { _.isDirectlyForceDependency } + } orElse (conflicts find { m => m.callers.exists { _.isForceDependency } - } match { + })) match { case Some(m) => + log.debug(s"- forced dependency: $m") (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString) case None => val strategy = lcm.getStrategy diff --git a/ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala b/ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala new file mode 100644 index 000000000..0a3338fac --- /dev/null +++ b/ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala @@ -0,0 +1,9 @@ +package sbt +package ivyint + +import org.apache.ivy.core +import core.module.descriptor.DefaultDependencyDescriptor + +trait SbtDefaultDependencyDescriptor { self: DefaultDependencyDescriptor => + def dependencyModuleId: ModuleID +}