From 54032ddd5fd50f270807b268f24c4b4cf4fbd603 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 8 Oct 2014 17:14:06 -0400 Subject: [PATCH 1/7] Fixes #1649. Adds exclusion rules into SHA-1 of minigraph path --- .../CachedResolutionResolveEngine.scala | 21 ++++++++- .../cached-resolution3/multi.sbt | 45 +++++++++++++++++++ .../cached-resolution3/test | 5 +++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 sbt/src/sbt-test/dependency-management/cached-resolution3/multi.sbt create mode 100644 sbt/src/sbt-test/dependency-management/cached-resolution3/test diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 4b6cd3e9e..3ee26964e 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -11,7 +11,7 @@ import org.apache.ivy.core import core.resolve._ import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport } -import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration } +import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule } import core.{ IvyPatternHelper, LogOptions } import org.apache.ivy.util.Message import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo } @@ -59,11 +59,28 @@ private[sbt] class CachedResolutionResolveCache() { } def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) = { + def excludeRuleString(rule: ExcludeRule): String = + s"""Exclude(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" + def includeRuleString(rule: IncludeRule): String = + s"""Include(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})""" val mrid = dd.getDependencyRevisionId val confMap = (dd.getModuleConfigurations map { conf => conf + "->(" + dd.getDependencyConfigurations(conf).mkString(",") + ")" }) - val depsString = mrid.toString + ";" + confMap.mkString(";") + val exclusions = (dd.getModuleConfigurations.toVector flatMap { conf => + dd.getExcludeRules(conf).toVector match { + case Vector() => None + case rules => Some(conf + "->(" + (rules map excludeRuleString).mkString(",") + ")") + } + }) + val inclusions = (dd.getModuleConfigurations.toVector flatMap { conf => + dd.getIncludeRules(conf).toVector match { + case Vector() => None + case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")") + } + }) + val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" + + s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};""" val sha1 = Hash.toHex(Hash(depsString)) val md1 = new DefaultModuleDescriptor(createID(sbtOrgTemp, "temp-resolve-" + sha1, "1.0"), "release", null, false) with ArtificialModuleDescriptor { def targetModuleRevisionId: ModuleRevisionId = mrid diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution3/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution3/multi.sbt new file mode 100644 index 000000000..3d3196156 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution3/multi.sbt @@ -0,0 +1,45 @@ +// https://github.com/sbt/sbt/issues/1649 +lazy val check = taskKey[Unit]("Runs the check") + +def commonSettings: Seq[Def.Setting[_]] = + Seq( + ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")), + dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency", + scalaVersion := "2.10.4", + resolvers += Resolver.sonatypeRepo("snapshots") + ) + +def cachedResolutionSettings: Seq[Def.Setting[_]] = + commonSettings ++ Seq( + updateOptions := updateOptions.value.withCachedResolution(true) + ) + +lazy val a = project. + settings(cachedResolutionSettings: _*). + settings( + libraryDependencies += "net.databinder" %% "unfiltered-uploads" % "0.8.0" exclude("commons-io", "commons-io") + ) + +lazy val b = project. + settings(cachedResolutionSettings: _*). + settings( + libraryDependencies += "net.databinder" %% "unfiltered-uploads" % "0.8.0" + ) + +lazy val root = (project in file(".")). + aggregate(a, b). + settings( + organization in ThisBuild := "org.example", + version in ThisBuild := "1.0", + check := { + // sys.error(dependencyCacheDirectory.value.toString) + val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName} + val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName} + if (acp exists { _.data.getName contains "commons-io" }) { + sys.error("commons-io found when it should be excluded") + } + if (!(bcp exists { _.data.getName contains "commons-io" })) { + sys.error("commons-io NOT found when it should NOT be excluded") + } + } + ) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution3/test b/sbt/src/sbt-test/dependency-management/cached-resolution3/test new file mode 100644 index 000000000..081d79708 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/cached-resolution3/test @@ -0,0 +1,5 @@ +> a/update + +> b/update + +> check From 09bca754b5b0a91eb627274638e74fe47c33c3c8 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 9 Oct 2014 13:11:13 -0400 Subject: [PATCH 2/7] Fixes #1639. Fixes cached resolution interacting with force() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When conflicts are found for a given module, a forced one is selected before conflict manager kicks in. The problem is that DependencyDescriptor seems to mark transitive forced dependency as forced as well, so the actual forced dependency are sometimes not prioritized. To work around this, I’ve introduced a mixin called SbtDefaultDependencyDescriptor, which carries around ModuleID to detect direct dependencies. --- ivy/src/main/scala/sbt/Ivy.scala | 6 ++-- ivy/src/main/scala/sbt/IvyRetrieve.scala | 12 ++++--- ivy/src/main/scala/sbt/UpdateReport.scala | 3 +- .../CachedResolutionResolveEngine.scala | 11 +++++-- .../SbtDefaultDependencyDescriptor.scala | 9 ++++++ .../actions/src/main/scala/sbt/CacheIvy.scala | 4 +-- .../cached-resolution2/multi.sbt | 32 +++++++++++++++++-- 7 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 4bf6ffad6..c0e7d508a 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -4,7 +4,7 @@ package sbt import Resolver.PluginPattern -import ivyint.{ CachedResolutionResolveEngine, CachedResolutionResolveCache } +import ivyint.{ CachedResolutionResolveEngine, CachedResolutionResolveCache, SbtDefaultDependencyDescriptor } import java.io.File import java.net.URI @@ -570,7 +570,9 @@ private[sbt] object IvySbt { /** Transforms an sbt ModuleID into an Ivy DefaultDependencyDescriptor.*/ def convertDependency(moduleID: DefaultModuleDescriptor, dependency: ModuleID, parser: CustomXmlParser.CustomParser): DefaultDependencyDescriptor = { - val dependencyDescriptor = new DefaultDependencyDescriptor(moduleID, toID(dependency), dependency.isForce, dependency.isChanging, dependency.isTransitive) + val dependencyDescriptor = new DefaultDependencyDescriptor(moduleID, toID(dependency), dependency.isForce, dependency.isChanging, dependency.isTransitive) with SbtDefaultDependencyDescriptor { + def dependencyModuleId = dependency + } dependency.configurations match { case None => // The configuration for this dependency was not explicitly specified, so use the default parser.parseDepsConfs(parser.getDefaultConf, dependencyDescriptor) diff --git a/ivy/src/main/scala/sbt/IvyRetrieve.scala b/ivy/src/main/scala/sbt/IvyRetrieve.scala index 1e74f3955..f453efb1f 100644 --- a/ivy/src/main/scala/sbt/IvyRetrieve.scala +++ b/ivy/src/main/scala/sbt/IvyRetrieve.scala @@ -13,6 +13,7 @@ import module.id.{ ModuleRevisionId, ModuleId => IvyModuleId } import report.{ ArtifactDownloadReport, ConfigurationResolveReport, ResolveReport } import resolve.{ IvyNode, IvyNodeCallers } import IvyNodeCallers.{ Caller => IvyCaller } +import ivyint.SbtDefaultDependencyDescriptor object IvyRetrieve { def reports(report: ResolveReport): Seq[ConfigurationResolveReport] = @@ -77,11 +78,14 @@ object IvyRetrieve { case x if nonEmptyString(x).isDefined => x } val ddOpt = Option(caller.getDependencyDescriptor) - val (extraAttributes, isForce, isChanging, isTransitive) = ddOpt match { - case Some(dd) => (toExtraAttributes(dd.getExtraAttributes), dd.isForce, dd.isChanging, dd.isTransitive) - case None => (Map.empty[String, String], false, false, true) + val (extraAttributes, isForce, isChanging, isTransitive, isDirectlyForce) = ddOpt match { + case Some(dd: SbtDefaultDependencyDescriptor) => + val mod = dd.dependencyModuleId + (toExtraAttributes(dd.getExtraAttributes), mod.isForce, mod.isChanging, mod.isTransitive, mod.isForce) + case Some(dd) => (toExtraAttributes(dd.getExtraAttributes), dd.isForce, dd.isChanging, dd.isTransitive, false) + case None => (Map.empty[String, String], false, false, true, false) } - new Caller(m, callerConfigurations, extraAttributes, isForce, isChanging, isTransitive) + new Caller(m, callerConfigurations, extraAttributes, isForce, isChanging, isTransitive, isDirectlyForce) } val revId = dep.getResolvedId val moduleId = toModuleID(revId) diff --git a/ivy/src/main/scala/sbt/UpdateReport.scala b/ivy/src/main/scala/sbt/UpdateReport.scala index 443c84ed3..696fa4d20 100644 --- a/ivy/src/main/scala/sbt/UpdateReport.scala +++ b/ivy/src/main/scala/sbt/UpdateReport.scala @@ -190,7 +190,8 @@ final class Caller( val callerExtraAttributes: Map[String, String], val isForceDependency: Boolean, val isChangingDependency: Boolean, - val isTransitiveDependency: Boolean) { + val isTransitiveDependency: Boolean, + val isDirectlyForceDependency: Boolean) { override def toString: String = s"$caller" } diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 3ee26964e..850f1b61d 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -36,6 +36,9 @@ private[sbt] class CachedResolutionResolveCache() { val mds = if (mrid0.getOrganisation == sbtOrgTemp) Vector(md0) else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 } + + updateReportCache.remove(md0.getModuleRevisionId) + directDependencyCache.remove(md0.getModuleRevisionId) mds foreach { md => updateReportCache.remove(md.getModuleRevisionId) directDependencyCache.remove(md.getModuleRevisionId) @@ -186,6 +189,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val miniGraphPath = depDir / "module" val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) val cache = cachedResolutionResolveCache + cache.directDependencyCache.remove(md0.getModuleRevisionId) val mds = cache.buildArtificialModuleDescriptors(md0, projectResolver) def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] = { @@ -292,10 +296,13 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val name = head.module.name log.debug(s"- conflict in $rootModuleConf:$organization:$name " + (conflicts map { _.module }).mkString("(", ", ", ")")) def useLatest(lcm: LatestConflictManager): (Vector[ModuleReport], Vector[ModuleReport], String) = - conflicts find { m => + (conflicts find { m => + m.callers.exists { _.isDirectlyForceDependency } + } orElse (conflicts find { m => m.callers.exists { _.isForceDependency } - } match { + })) match { case Some(m) => + log.debug(s"- forced dependency: $m") (Vector(m), conflicts filterNot { _ == m } map { _.copy(evicted = true, evictedReason = Some(lcm.toString)) }, lcm.toString) case None => val strategy = lcm.getStrategy diff --git a/ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala b/ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala new file mode 100644 index 000000000..0a3338fac --- /dev/null +++ b/ivy/src/main/scala/sbt/ivyint/SbtDefaultDependencyDescriptor.scala @@ -0,0 +1,9 @@ +package sbt +package ivyint + +import org.apache.ivy.core +import core.module.descriptor.DefaultDependencyDescriptor + +trait SbtDefaultDependencyDescriptor { self: DefaultDependencyDescriptor => + def dependencyModuleId: ModuleID +} diff --git a/main/actions/src/main/scala/sbt/CacheIvy.scala b/main/actions/src/main/scala/sbt/CacheIvy.scala index 17e260450..91e82107e 100644 --- a/main/actions/src/main/scala/sbt/CacheIvy.scala +++ b/main/actions/src/main/scala/sbt/CacheIvy.scala @@ -81,8 +81,8 @@ object CacheIvy { implicit def organizationArtifactReportFormat(implicit sf: Format[String], bf: Format[Boolean], df: Format[Seq[ModuleReport]]): Format[OrganizationArtifactReport] = wrap[OrganizationArtifactReport, (String, String, Seq[ModuleReport])](m => (m.organization, m.name, m.modules), { case (o, n, r) => OrganizationArtifactReport(o, n, r) }) implicit def callerFormat: Format[Caller] = - wrap[Caller, (ModuleID, Seq[String], Map[String, String], Boolean, Boolean, Boolean)](c => (c.caller, c.callerConfigurations, c.callerExtraAttributes, c.isForceDependency, c.isChangingDependency, c.isTransitiveDependency), - { case (c, cc, ea, fd, cd, td) => new Caller(c, cc, ea, fd, cd, td) }) + wrap[Caller, (ModuleID, Seq[String], Map[String, String], Boolean, Boolean, Boolean, Boolean)](c => (c.caller, c.callerConfigurations, c.callerExtraAttributes, c.isForceDependency, c.isChangingDependency, c.isTransitiveDependency, c.isDirectlyForceDependency), + { case (c, cc, ea, fd, cd, td, df) => new Caller(c, cc, ea, fd, cd, td, df) }) implicit def exclusionRuleFormat(implicit sf: Format[String]): Format[ExclusionRule] = wrap[ExclusionRule, (String, String, String, Seq[String])](e => (e.organization, e.name, e.artifact, e.configurations), { case (o, n, a, cs) => ExclusionRule(o, n, a, cs) }) implicit def crossVersionFormat: Format[CrossVersion] = wrap(crossToInt, crossFromInt) diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt b/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt index e106a7693..a3c3d5a87 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt +++ b/sbt/src/sbt-test/dependency-management/cached-resolution2/multi.sbt @@ -5,8 +5,10 @@ def commonSettings: Seq[Def.Setting[_]] = ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")), dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency", libraryDependencies := Seq( - "org.springframework" % "spring-core" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), - "org.springframework" % "spring-context" % "4.0.3.RELEASE" exclude("org.springframework", "spring-asm") + "org.springframework" % "spring-core" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-tx" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-beans" % "3.2.2.RELEASE" force() exclude("org.springframework", "spring-asm"), + "org.springframework" % "spring-context" % "3.1.2.RELEASE" force() exclude("org.springframework", "spring-asm") ), scalaVersion := "2.10.4", resolvers += Resolver.sonatypeRepo("snapshots") @@ -25,7 +27,12 @@ lazy val b = project. lazy val c = project. dependsOn(a). - settings(cachedResolutionSettings: _*) + settings(cachedResolutionSettings: _*). + settings( + libraryDependencies := Seq( + "org.springframework" % "spring-core" % "4.0.3.RELEASE" exclude("org.springframework", "spring-asm") + ) + ) lazy val root = (project in file(".")). aggregate(a, b, c). @@ -37,6 +44,25 @@ lazy val root = (project in file(".")). val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName} val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName} val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} + + if (!(acp exists {_.data.getName contains "spring-core-3.2.2.RELEASE"})) { + sys.error("spring-core-3.2.2 is not found") + } + if (!(bcp exists {_.data.getName contains "spring-core-3.2.2.RELEASE"})) { + sys.error("spring-core-3.2.2 is not found") + } + if (!(ccp exists {_.data.getName contains "spring-core-3.2.2.RELEASE"})) { + sys.error("spring-core-3.2.2 is not found") + } + if (!(acp exists {_.data.getName contains "spring-tx-3.1.2.RELEASE"})) { + sys.error("spring-tx-3.1.2 is not found") + } + if (!(bcp exists {_.data.getName contains "spring-tx-3.1.2.RELEASE"})) { + sys.error("spring-tx-3.1.2 is not found") + } + if (!(ccp exists {_.data.getName contains "spring-tx-3.1.2.RELEASE"})) { + sys.error("spring-tx-3.1.2 is not found") + } if (acp == bcp && acp == ccp) () else sys.error("Different classpaths are found:" + "\n - a (consolidated) " + acp.toString + From 8ddff28a9e0b564503b71b795ac24873b294e16b Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Oct 2014 00:34:05 -0400 Subject: [PATCH 3/7] Fixes #1660 Provide overloads for transitiveScratch and updateClassifiers that work with IvyActions.updateEither. --- ivy/src/main/scala/sbt/IvyActions.scala | 24 ++++++++++-- ivy/src/main/scala/sbt/LogicalClock.scala | 5 +++ main/src/main/scala/sbt/Defaults.scala | 38 +++++++++++++------ .../cached-resolution/test | 2 + 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/ivy/src/main/scala/sbt/IvyActions.scala b/ivy/src/main/scala/sbt/IvyActions.scala index f5e57336d..d52bf23d6 100644 --- a/ivy/src/main/scala/sbt/IvyActions.scala +++ b/ivy/src/main/scala/sbt/IvyActions.scala @@ -198,17 +198,31 @@ object IvyActions { def grouped[T](grouping: ModuleID => T)(mods: Seq[ModuleID]): Map[T, Set[String]] = mods groupBy (grouping) mapValues (_.map(_.revision).toSet) + @deprecated("This is no longer public.", "0.13.6") def transitiveScratch(ivySbt: IvySbt, label: String, config: GetClassifiersConfiguration, log: Logger): UpdateReport = + transitiveScratch(ivySbt, label, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) + + private[sbt] def transitiveScratch(ivySbt: IvySbt, label: String, config: GetClassifiersConfiguration, + uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): UpdateReport = { import config.{ configuration => c, ivyScala, module => mod } import mod.{ id, modules => deps } val base = restrictedCopy(id, true).copy(name = id.name + "$" + label) val module = new ivySbt.Module(InlineConfiguration(base, ModuleInfo(base.name), deps).copy(ivyScala = ivyScala)) - val report = update(module, c, log) + val report = updateEither(module, c, uwconfig, logicalClock, depDir, log) match { + case Right(r) => r + case Left(w) => + throw w.resolveException + } val newConfig = config.copy(module = mod.copy(modules = report.allModules)) - updateClassifiers(ivySbt, newConfig, log) + updateClassifiers(ivySbt, newConfig, uwconfig, logicalClock, depDir, log) } + @deprecated("This is no longer public.", "0.13.6") def updateClassifiers(ivySbt: IvySbt, config: GetClassifiersConfiguration, log: Logger): UpdateReport = + updateClassifiers(ivySbt, config, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) + + private[sbt] def updateClassifiers(ivySbt: IvySbt, config: GetClassifiersConfiguration, + uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): UpdateReport = { import config.{ configuration => c, module => mod, _ } import mod.{ configurations => confs, _ } @@ -218,7 +232,11 @@ object IvyActions { val base = restrictedCopy(id, true).copy(name = id.name + classifiers.mkString("$", "_", "")) val module = new ivySbt.Module(InlineConfiguration(base, ModuleInfo(base.name), deps).copy(ivyScala = ivyScala, configurations = confs)) val upConf = new UpdateConfiguration(c.retrieve, true, c.logging) - update(module, upConf, log) + updateEither(module, upConf, uwconfig, logicalClock, depDir, log) match { + case Right(r) => r + case Left(w) => + throw w.resolveException + } } def classifiedArtifacts(classifiers: Seq[String], exclude: Map[ModuleID, Set[String]])(m: ModuleID): Option[ModuleID] = { diff --git a/ivy/src/main/scala/sbt/LogicalClock.scala b/ivy/src/main/scala/sbt/LogicalClock.scala index 1056f6560..d4e1d4cc3 100644 --- a/ivy/src/main/scala/sbt/LogicalClock.scala +++ b/ivy/src/main/scala/sbt/LogicalClock.scala @@ -9,6 +9,11 @@ trait LogicalClock { } object LogicalClock { + def apply(hashCode: Int): LogicalClock = { + def intToByteArray(x: Int): Array[Byte] = + Array((x >>> 24).toByte, (x >> 16 & 0xff).toByte, (x >> 8 & 0xff).toByte, (x & 0xff).toByte) + apply(Hash.toHex(intToByteArray(hashCode))) + } def apply(x: String): LogicalClock = new LogicalClock { override def toString: String = x } diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 077152859..91b888538 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -1135,10 +1135,20 @@ object Classpaths { val externalModules = update.value.allModules.filterNot(m => projectDeps contains key(m)) GetClassifiersModule(projectID.value, externalModules, ivyConfigurations.in(updateClassifiers).value, transitiveClassifiers.in(updateClassifiers).value) }, - updateClassifiers <<= (ivySbt, classifiersModule in updateClassifiers, updateConfiguration, ivyScala, appConfiguration, streams) map { (is, mod, c, ivyScala, app, s) => + updateClassifiers <<= Def.task { + val s = streams.value + val is = ivySbt.value + val mod = (classifiersModule in updateClassifiers).value + val c = updateConfiguration.value + val app = appConfiguration.value val out = is.withIvy(s.log)(_.getSettings.getDefaultIvyUserDir) + val uwConfig = (unresolvedWarningConfiguration in update).value + val depDir = dependencyCacheDirectory.value withExcludes(out, mod.classifiers, lock(app)) { excludes => - IvyActions.updateClassifiers(is, GetClassifiersConfiguration(mod, excludes, c, ivyScala), s.log) + val uwConfig = (unresolvedWarningConfiguration in update).value + val logicalClock = LogicalClock(state.value.hashCode) + val depDir = dependencyCacheDirectory.value + IvyActions.updateClassifiers(is, GetClassifiersConfiguration(mod, excludes, c, ivyScala.value), uwConfig, LogicalClock(state.value.hashCode), Some(depDir), s.log) } } tag (Tags.Update, Tags.Network) ) @@ -1203,13 +1213,19 @@ object Classpaths { val pluginIDs: Seq[ModuleID] = pluginJars.flatMap(_ get moduleID.key) GetClassifiersModule(pid, sbtDep +: pluginIDs, Configurations.Default :: Nil, classifiers) }, - updateSbtClassifiers in TaskGlobal <<= (ivySbt, classifiersModule, updateConfiguration, ivyScala, appConfiguration, streams) map { - (is, mod, c, ivyScala, app, s) => - val out = is.withIvy(s.log)(_.getSettings.getDefaultIvyUserDir) - withExcludes(out, mod.classifiers, lock(app)) { excludes => - val noExplicitCheck = ivyScala.map(_.copy(checkExplicit = false)) - IvyActions.transitiveScratch(is, "sbt", GetClassifiersConfiguration(mod, excludes, c, noExplicitCheck), s.log) - } + updateSbtClassifiers in TaskGlobal <<= Def.task { + val s = streams.value + val is = ivySbt.value + val mod = classifiersModule.value + val c = updateConfiguration.value + val app = appConfiguration.value + val out = is.withIvy(s.log)(_.getSettings.getDefaultIvyUserDir) + val uwConfig = (unresolvedWarningConfiguration in update).value + val depDir = dependencyCacheDirectory.value + withExcludes(out, mod.classifiers, lock(app)) { excludes => + val noExplicitCheck = ivyScala.value.map(_.copy(checkExplicit = false)) + IvyActions.transitiveScratch(is, "sbt", GetClassifiersConfiguration(mod, excludes, c, noExplicitCheck), uwConfig, LogicalClock(state.value.hashCode), Some(depDir), s.log) + } } tag (Tags.Update, Tags.Network) )) @@ -1259,8 +1275,6 @@ object Classpaths { case (Some(res), _, Some(decl)) if res == decl => jars case _ => Nil } - def intToByteArray(x: Int): Array[Byte] = - Array((x >>> 24).toByte, (x >> 16 & 0xff).toByte, (x >> 8 & 0xff).toByte, (x & 0xff).toByte) val subScalaJars: String => Seq[File] = Defaults.unmanagedScalaInstanceOnly.value match { case Some(si) => subUnmanaged(si.version, si.jars) case None => sv => if (scalaProvider.version == sv) scalaProvider.jars else Nil @@ -1269,7 +1283,7 @@ object Classpaths { val uwConfig = (unresolvedWarningConfiguration in update).value val show = Reference.display(thisProjectRef.value) val st = state.value - val logicalClock = LogicalClock(Hash.toHex(intToByteArray(st.hashCode))) + val logicalClock = LogicalClock(st.hashCode) val depDir = dependencyCacheDirectory.value cachedUpdate(s.cacheDirectory / updateCacheName.value, show, ivyModule.value, updateConfiguration.value, transform, skip = (skip in update).value, force = isRoot, depsUpdated = depsUpdated, diff --git a/sbt/src/sbt-test/dependency-management/cached-resolution/test b/sbt/src/sbt-test/dependency-management/cached-resolution/test index bd1a58183..684215c92 100644 --- a/sbt/src/sbt-test/dependency-management/cached-resolution/test +++ b/sbt/src/sbt-test/dependency-management/cached-resolution/test @@ -1,3 +1,5 @@ +> a/updateClassifiers + > a/publishLocal > check From be763ad65542aa97e48e042da9c3ab841050ff32 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Oct 2014 03:20:12 -0400 Subject: [PATCH 4/7] Make sure IvyAction.update works --- ivy/src/main/scala/sbt/IvyActions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/IvyActions.scala b/ivy/src/main/scala/sbt/IvyActions.scala index d52bf23d6..41850b247 100644 --- a/ivy/src/main/scala/sbt/IvyActions.scala +++ b/ivy/src/main/scala/sbt/IvyActions.scala @@ -155,7 +155,7 @@ object IvyActions { private[sbt] def updateEither(module: IvySbt#Module, configuration: UpdateConfiguration, uwconfig: UnresolvedWarningConfiguration, logicalClock: LogicalClock, depDir: Option[File], log: Logger): Either[UnresolvedWarning, UpdateReport] = module.withModule(log) { - case (ivy, md, default) if module.owner.configuration.updateOptions.cachedResolution => + case (ivy, md, default) if module.owner.configuration.updateOptions.cachedResolution && depDir.isDefined => ivy.getResolveEngine match { case x: CachedResolutionResolveEngine => val resolveOptions = new ResolveOptions From b8873614e02a0da8ff151770c0d7e7f7f76282e7 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Oct 2014 06:40:56 -0400 Subject: [PATCH 5/7] Use correct stats for cached resolution --- ivy/src/main/scala/sbt/UpdateReport.scala | 11 +++++++++++ .../ivyint/CachedResolutionResolveEngine.scala | 18 ++++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/ivy/src/main/scala/sbt/UpdateReport.scala b/ivy/src/main/scala/sbt/UpdateReport.scala index 696fa4d20..03ee2fe31 100644 --- a/ivy/src/main/scala/sbt/UpdateReport.scala +++ b/ivy/src/main/scala/sbt/UpdateReport.scala @@ -34,6 +34,12 @@ final class UpdateReport(val cachedDescriptor: File, val configurations: Seq[Con /** Gets the names of all resolved configurations. This `UpdateReport` contains one `ConfigurationReport` for each configuration in this list. */ def allConfigurations: Seq[String] = configurations.map(_.configuration) + + private[sbt] def withStats(us: UpdateStats): UpdateReport = + new UpdateReport(this.cachedDescriptor, + this.configurations, + us, + this.stamps) } /** @@ -268,4 +274,9 @@ object UpdateReport { } final class UpdateStats(val resolveTime: Long, val downloadTime: Long, val downloadSize: Long, val cached: Boolean) { override def toString = Seq("Resolve time: " + resolveTime + " ms", "Download time: " + downloadTime + " ms", "Download size: " + downloadSize + " bytes").mkString(", ") + private[sbt] def withCached(c: Boolean): UpdateStats = + new UpdateStats(resolveTime = this.resolveTime, + downloadTime = this.downloadTime, + downloadSize = this.downloadSize, + cached = c) } \ No newline at end of file diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index 850f1b61d..e57499737 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -119,7 +119,11 @@ private[sbt] class CachedResolutionResolveCache() { Right(ur) } (updateReportCache.get(mrid) orElse loadMiniGraphFromFile) match { - case Some(result) => result + case Some(result) => + result match { + case Right(ur) => Right(ur.withStats(ur.stats.withCached(true))) + case x => x + } case None => f match { case Right(ur) => @@ -186,6 +190,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { // Return sbt's UpdateReport. def customResolve(md0: ModuleDescriptor, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { import Path._ + val start = System.currentTimeMillis val miniGraphPath = depDir / "module" val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) val cache = cachedResolutionResolveCache @@ -217,7 +222,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { doWork(md) } } - val uReport = mergeResults(md0, results, log) + val uReport = mergeResults(md0, results, System.currentTimeMillis - start, log) val cacheManager = getSettings.getResolutionCacheManager cacheManager.saveResolvedModuleDescriptor(md0) val prop0 = "" @@ -225,9 +230,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { IO.write(ivyPropertiesInCache0, prop0) uReport } - def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], log: Logger): Either[ResolveException, UpdateReport] = + def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], resolveTime: Long, log: Logger): Either[ResolveException, UpdateReport] = if (results exists { _.isLeft }) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) - else Right(mergeReports(md0, results collect { case Right(ur) => ur }, log)) + else Right(mergeReports(md0, results collect { case Right(ur) => ur }, resolveTime, log)) def mergeErrors(md0: ModuleDescriptor, errors: Vector[ResolveException], log: Logger): ResolveException = { val messages = errors flatMap { _.messages } @@ -241,11 +246,12 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { } new ResolveException(messages, failed, ListMap(failedPaths: _*)) } - def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], log: Logger): UpdateReport = + def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], resolveTime: Long, log: Logger): UpdateReport = { val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId) val rootModuleConfigs = md0.getConfigurations.toVector - val stats = new UpdateStats(0L, 0L, 0L, false) + val cachedReports = reports filter { !_.stats.cached } + val stats = new UpdateStats(resolveTime, (cachedReports map { _.stats.downloadTime }).sum, (cachedReports map { _.stats.downloadSize }).sum, false) val configReports = rootModuleConfigs map { conf => val crs = reports flatMap { _.configurations filter { _.configuration == conf.getName } } mergeConfigurationReports(conf.getName, crs, log) From 1043d027d0c688f09b8e0aceec31b1982adc83fd Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Oct 2014 06:50:36 -0400 Subject: [PATCH 6/7] Make sure all files exist in minigraph. --- .../ivyint/CachedResolutionResolveEngine.scala | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index e57499737..e7ef58b42 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -112,11 +112,19 @@ private[sbt] class CachedResolutionResolveCache() { def loadMiniGraphFromFile: Option[Either[ResolveException, UpdateReport]] = (if (staticGraphPath.exists) Some(staticGraphPath) else if (dynamicGraphPath.exists) Some(dynamicGraphPath) - else None) map { path => - log.debug(s"parsing ${path.getAbsolutePath.toString}") - val ur = JsonUtil.parseUpdateReport(md, path, cachedDescriptor, log) - updateReportCache(md.getModuleRevisionId) = Right(ur) - Right(ur) + else None) match { + case Some(path) => + log.debug(s"parsing ${path.getAbsolutePath.toString}") + val ur = JsonUtil.parseUpdateReport(md, path, cachedDescriptor, log) + if (ur.allFiles forall { _.exists }) { + updateReportCache(md.getModuleRevisionId) = Right(ur) + Some(Right(ur)) + } else { + log.debug(s"some files are missing from the cache, so invalidating the minigraph") + IO.delete(path) + None + } + case _ => None } (updateReportCache.get(mrid) orElse loadMiniGraphFromFile) match { case Some(result) => From c47cbf5d02322c558256a77e2f05ba953bc0ccad Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Fri, 10 Oct 2014 17:25:43 -0400 Subject: [PATCH 7/7] Implement configuration.missingOk to fix updateClassifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updateClassifiers sets missingOk to true. cached resolution wasn’t respecting the looseness. --- ivy/src/main/scala/sbt/IvyActions.scala | 2 +- .../ivyint/CachedResolutionResolveEngine.scala | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ivy/src/main/scala/sbt/IvyActions.scala b/ivy/src/main/scala/sbt/IvyActions.scala index 41850b247..7dd07df7d 100644 --- a/ivy/src/main/scala/sbt/IvyActions.scala +++ b/ivy/src/main/scala/sbt/IvyActions.scala @@ -161,7 +161,7 @@ object IvyActions { val resolveOptions = new ResolveOptions val resolveId = ResolveOptions.getDefaultResolveId(md) resolveOptions.setResolveId(resolveId) - x.customResolve(md, logicalClock, resolveOptions, depDir getOrElse { sys.error("dependency base directory is not specified") }, log) match { + x.customResolve(md, configuration.missingOk, logicalClock, resolveOptions, depDir getOrElse { sys.error("dependency base directory is not specified") }, log) match { case Left(x) => Left(UnresolvedWarning(x, uwconfig)) case Right(uReport) => diff --git a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala index e7ef58b42..09da3b501 100644 --- a/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala +++ b/ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala @@ -195,8 +195,11 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { private[sbt] def projectResolver: Option[ProjectResolver] private[sbt] def makeInstance: Ivy - // Return sbt's UpdateReport. - def customResolve(md0: ModuleDescriptor, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { + /** + * This returns sbt's UpdateReport structure. + * missingOk allows sbt to call this with classifiers that may or may not exist, and grab the JARs. + */ + def customResolve(md0: ModuleDescriptor, missingOk: Boolean, logicalClock: LogicalClock, options0: ResolveOptions, depDir: File, log: Logger): Either[ResolveException, UpdateReport] = { import Path._ val start = System.currentTimeMillis val miniGraphPath = depDir / "module" @@ -209,7 +212,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { val options1 = new ResolveOptions(options0) val i = makeInstance var rr = i.resolve(md, options1) - if (!rr.hasError) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) + if (!rr.hasError || missingOk) Right(IvyRetrieve.updateReport(rr, cachedDescriptor)) else { val messages = rr.getAllProblemMessages.toArray.map(_.toString).distinct val failedPaths = ListMap(rr.getUnresolvedDependencies map { node => @@ -230,7 +233,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { doWork(md) } } - val uReport = mergeResults(md0, results, System.currentTimeMillis - start, log) + val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, log) val cacheManager = getSettings.getResolutionCacheManager cacheManager.saveResolvedModuleDescriptor(md0) val prop0 = "" @@ -238,8 +241,8 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine { IO.write(ivyPropertiesInCache0, prop0) uReport } - def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], resolveTime: Long, log: Logger): Either[ResolveException, UpdateReport] = - if (results exists { _.isLeft }) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) + def mergeResults(md0: ModuleDescriptor, results: Vector[Either[ResolveException, UpdateReport]], missingOk: Boolean, resolveTime: Long, log: Logger): Either[ResolveException, UpdateReport] = + if (!missingOk && (results exists { _.isLeft })) Left(mergeErrors(md0, results collect { case Left(re) => re }, log)) else Right(mergeReports(md0, results collect { case Right(ur) => ur }, resolveTime, log)) def mergeErrors(md0: ModuleDescriptor, errors: Vector[ResolveException], log: Logger): ResolveException = {